2024-11-17 07:13:28,890 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 07:13:28,905 main DEBUG Took 0.012619 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 07:13:28,905 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 07:13:28,906 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 07:13:28,907 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 07:13:28,909 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 07:13:28,918 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 07:13:28,929 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,931 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 07:13:28,932 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,933 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 07:13:28,933 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,934 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 07:13:28,935 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,935 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 07:13:28,936 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,936 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 07:13:28,937 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 07:13:28,939 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,939 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 07:13:28,940 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 07:13:28,941 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 07:13:28,942 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 07:13:28,943 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,943 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 07:13:28,944 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 07:13:28,945 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,945 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 07:13:28,947 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 07:13:28,949 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 07:13:28,951 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 07:13:28,952 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 07:13:28,953 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 07:13:28,954 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 07:13:28,965 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 07:13:28,968 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 07:13:28,971 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 07:13:28,971 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 07:13:28,971 main DEBUG createAppenders(={Console}) 2024-11-17 07:13:28,972 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-17 07:13:28,972 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-17 07:13:28,973 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-17 07:13:28,973 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 07:13:28,974 main DEBUG OutputStream closed 2024-11-17 07:13:28,974 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 07:13:28,974 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 07:13:28,974 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-17 07:13:29,053 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 07:13:29,056 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 07:13:29,057 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 07:13:29,059 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 07:13:29,059 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 07:13:29,060 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 07:13:29,060 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 07:13:29,061 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 07:13:29,061 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 07:13:29,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 07:13:29,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 07:13:29,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 07:13:29,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 07:13:29,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 07:13:29,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 07:13:29,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 07:13:29,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 07:13:29,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 07:13:29,069 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 07:13:29,069 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-17 07:13:29,070 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 07:13:29,071 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-17T07:13:29,334 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d 2024-11-17 07:13:29,337 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 07:13:29,337 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T07:13:29,346 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-17T07:13:29,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-17T07:13:29,378 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/cluster_ec49d093-dc1d-39e4-b51d-2529c25f0961, deleteOnExit=true 2024-11-17T07:13:29,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-17T07:13:29,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/test.cache.data in system properties and HBase conf 2024-11-17T07:13:29,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/hadoop.tmp.dir in system properties and HBase conf 2024-11-17T07:13:29,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/hadoop.log.dir in system properties and HBase conf 2024-11-17T07:13:29,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-17T07:13:29,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-17T07:13:29,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-17T07:13:29,466 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T07:13:29,550 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T07:13:29,553 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-17T07:13:29,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-17T07:13:29,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-17T07:13:29,554 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T07:13:29,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-17T07:13:29,555 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-17T07:13:29,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-17T07:13:29,556 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T07:13:29,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-17T07:13:29,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/nfs.dump.dir in system properties and HBase conf 2024-11-17T07:13:29,557 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/java.io.tmpdir in system properties and HBase conf 2024-11-17T07:13:29,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-17T07:13:29,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-17T07:13:29,558 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-17T07:13:31,266 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-17T07:13:31,344 INFO [Time-limited test {}] log.Log(170): Logging initialized @3125ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-17T07:13:31,418 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T07:13:31,477 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T07:13:31,497 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T07:13:31,497 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T07:13:31,498 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-17T07:13:31,511 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T07:13:31,513 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/hadoop.log.dir/,AVAILABLE} 2024-11-17T07:13:31,514 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T07:13:31,750 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/java.io.tmpdir/jetty-localhost-39257-hadoop-hdfs-3_4_1-tests_jar-_-any-14209601881680304133/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T07:13:31,762 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:39257} 2024-11-17T07:13:31,763 INFO [Time-limited test {}] server.Server(415): Started @3545ms 2024-11-17T07:13:32,165 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-17T07:13:32,171 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-17T07:13:32,172 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-17T07:13:32,173 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-17T07:13:32,173 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-17T07:13:32,174 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/hadoop.log.dir/,AVAILABLE} 2024-11-17T07:13:32,174 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-17T07:13:32,293 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7acb84b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/java.io.tmpdir/jetty-localhost-35147-hadoop-hdfs-3_4_1-tests_jar-_-any-15257079477351512746/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T07:13:32,294 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:35147} 2024-11-17T07:13:32,294 INFO [Time-limited test {}] server.Server(415): Started @4076ms 2024-11-17T07:13:32,352 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-17T07:13:32,836 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/cluster_ec49d093-dc1d-39e4-b51d-2529c25f0961/data/data2/current/BP-711933474-172.17.0.2-1731827610116/current, will proceed with Du for space computation calculation, 2024-11-17T07:13:32,836 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/cluster_ec49d093-dc1d-39e4-b51d-2529c25f0961/data/data1/current/BP-711933474-172.17.0.2-1731827610116/current, will proceed with Du for space computation calculation, 2024-11-17T07:13:32,868 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-17T07:13:32,918 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa7b618a4b8f94aa0 with lease ID 0x677ac6945ae6ce9b: Processing first storage report for DS-2e1b16d1-e222-4f01-af4c-e06070bab600 from datanode DatanodeRegistration(127.0.0.1:46705, datanodeUuid=2c01d2b6-9a80-45f3-9f18-75dcd942ec6c, infoPort=43811, infoSecurePort=0, ipcPort=33895, storageInfo=lv=-57;cid=testClusterID;nsid=1140202542;c=1731827610116) 2024-11-17T07:13:32,921 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa7b618a4b8f94aa0 with lease ID 0x677ac6945ae6ce9b: from storage DS-2e1b16d1-e222-4f01-af4c-e06070bab600 node DatanodeRegistration(127.0.0.1:46705, datanodeUuid=2c01d2b6-9a80-45f3-9f18-75dcd942ec6c, infoPort=43811, infoSecurePort=0, ipcPort=33895, storageInfo=lv=-57;cid=testClusterID;nsid=1140202542;c=1731827610116), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-17T07:13:32,921 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa7b618a4b8f94aa0 with lease ID 0x677ac6945ae6ce9b: Processing first storage report for DS-0f210e3f-ae54-46b6-b572-86940c80edcb from datanode DatanodeRegistration(127.0.0.1:46705, datanodeUuid=2c01d2b6-9a80-45f3-9f18-75dcd942ec6c, infoPort=43811, infoSecurePort=0, ipcPort=33895, storageInfo=lv=-57;cid=testClusterID;nsid=1140202542;c=1731827610116) 2024-11-17T07:13:32,921 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa7b618a4b8f94aa0 with lease ID 0x677ac6945ae6ce9b: from storage DS-0f210e3f-ae54-46b6-b572-86940c80edcb node DatanodeRegistration(127.0.0.1:46705, datanodeUuid=2c01d2b6-9a80-45f3-9f18-75dcd942ec6c, infoPort=43811, infoSecurePort=0, ipcPort=33895, storageInfo=lv=-57;cid=testClusterID;nsid=1140202542;c=1731827610116), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-17T07:13:32,934 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d 2024-11-17T07:13:33,010 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/cluster_ec49d093-dc1d-39e4-b51d-2529c25f0961/zookeeper_0, clientPort=56533, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/cluster_ec49d093-dc1d-39e4-b51d-2529c25f0961/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/cluster_ec49d093-dc1d-39e4-b51d-2529c25f0961/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T07:13:33,020 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56533 2024-11-17T07:13:33,030 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T07:13:33,033 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T07:13:33,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741825_1001 (size=7) 2024-11-17T07:13:33,687 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650 with version=8 2024-11-17T07:13:33,687 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/hbase-staging 2024-11-17T07:13:33,811 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T07:13:34,119 INFO [Time-limited test {}] client.ConnectionUtils(128): master/1dd62e24b303:0 server-side Connection retries=45 2024-11-17T07:13:34,135 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T07:13:34,136 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T07:13:34,141 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T07:13:34,141 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T07:13:34,141 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T07:13:34,319 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T07:13:34,392 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T07:13:34,404 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T07:13:34,410 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T07:13:34,449 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3064 (auto-detected) 2024-11-17T07:13:34,450 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-17T07:13:34,479 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36491 2024-11-17T07:13:34,511 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36491 connecting to ZooKeeper ensemble=127.0.0.1:56533 2024-11-17T07:13:34,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:364910x0, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T07:13:34,590 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36491-0x10124c627cf0000 connected 2024-11-17T07:13:34,652 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T07:13:34,657 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T07:13:34,673 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T07:13:34,679 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650, hbase.cluster.distributed=false 2024-11-17T07:13:34,715 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T07:13:34,722 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36491 2024-11-17T07:13:34,722 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36491 2024-11-17T07:13:34,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36491 2024-11-17T07:13:34,729 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36491 2024-11-17T07:13:34,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36491 2024-11-17T07:13:34,855 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/1dd62e24b303:0 server-side Connection retries=45 2024-11-17T07:13:34,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T07:13:34,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T07:13:34,857 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T07:13:34,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T07:13:34,857 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T07:13:34,860 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T07:13:34,862 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T07:13:34,863 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36419 2024-11-17T07:13:34,865 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36419 connecting to ZooKeeper ensemble=127.0.0.1:56533 2024-11-17T07:13:34,866 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T07:13:34,869 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T07:13:34,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:364190x0, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T07:13:34,877 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36419-0x10124c627cf0001 connected 2024-11-17T07:13:34,878 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T07:13:34,882 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T07:13:34,890 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T07:13:34,892 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T07:13:34,897 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T07:13:34,897 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36419 2024-11-17T07:13:34,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36419 2024-11-17T07:13:34,898 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36419 2024-11-17T07:13:34,899 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36419 2024-11-17T07:13:34,899 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36419 2024-11-17T07:13:34,914 DEBUG [M:0;1dd62e24b303:36491 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;1dd62e24b303:36491 2024-11-17T07:13:34,915 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/1dd62e24b303,36491,1731827613877 2024-11-17T07:13:34,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T07:13:34,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T07:13:34,924 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/1dd62e24b303,36491,1731827613877 2024-11-17T07:13:34,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T07:13:34,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:13:34,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:13:34,944 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T07:13:34,945 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/1dd62e24b303,36491,1731827613877 from backup master directory 2024-11-17T07:13:34,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/1dd62e24b303,36491,1731827613877 2024-11-17T07:13:34,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T07:13:34,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T07:13:34,949 WARN [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T07:13:34,949 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=1dd62e24b303,36491,1731827613877 2024-11-17T07:13:34,952 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T07:13:34,954 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T07:13:35,010 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/hbase.id] with ID: d37c6a52-7167-4bf7-960c-03142c6fe8ff 2024-11-17T07:13:35,010 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/.tmp/hbase.id 2024-11-17T07:13:35,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741826_1002 (size=42) 2024-11-17T07:13:35,422 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/.tmp/hbase.id]:[hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/hbase.id] 2024-11-17T07:13:35,466 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-17T07:13:35,471 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T07:13:35,490 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-17T07:13:35,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:13:35,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:13:35,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741827_1003 (size=196) 2024-11-17T07:13:35,929 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T07:13:35,932 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T07:13:35,952 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:35,958 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T07:13:35,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741828_1004 (size=1189) 2024-11-17T07:13:36,415 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store 2024-11-17T07:13:36,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741829_1005 (size=34) 2024-11-17T07:13:36,839 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T07:13:36,843 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:13:36,845 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T07:13:36,845 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T07:13:36,845 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T07:13:36,847 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T07:13:36,847 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T07:13:36,848 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T07:13:36,849 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731827616844Disabling compacts and flushes for region at 1731827616844Disabling writes for close at 1731827616847 (+3 ms)Writing region close event to WAL at 1731827616848 (+1 ms)Closed at 1731827616848 2024-11-17T07:13:36,852 WARN [master/1dd62e24b303:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/.initializing 2024-11-17T07:13:36,852 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/WALs/1dd62e24b303,36491,1731827613877 2024-11-17T07:13:36,863 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T07:13:36,879 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1dd62e24b303%2C36491%2C1731827613877, suffix=, logDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/WALs/1dd62e24b303,36491,1731827613877, archiveDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/oldWALs, maxLogs=10 2024-11-17T07:13:36,910 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/WALs/1dd62e24b303,36491,1731827613877/1dd62e24b303%2C36491%2C1731827613877.1731827616885, exclude list is [], retry=0 2024-11-17T07:13:36,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46705,DS-2e1b16d1-e222-4f01-af4c-e06070bab600,DISK] 2024-11-17T07:13:36,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-17T07:13:36,968 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/WALs/1dd62e24b303,36491,1731827613877/1dd62e24b303%2C36491%2C1731827613877.1731827616885 2024-11-17T07:13:36,969 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43811:43811)] 2024-11-17T07:13:36,969 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T07:13:36,970 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:13:36,973 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T07:13:36,974 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T07:13:37,012 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T07:13:37,042 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T07:13:37,047 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:37,050 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T07:13:37,050 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T07:13:37,054 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T07:13:37,054 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:37,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:13:37,056 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T07:13:37,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T07:13:37,060 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:37,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:13:37,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T07:13:37,064 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T07:13:37,064 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:37,069 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:13:37,069 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T07:13:37,073 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T07:13:37,074 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T07:13:37,080 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T07:13:37,080 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T07:13:37,084 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T07:13:37,087 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T07:13:37,091 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T07:13:37,093 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73064046, jitterRate=0.08873912692070007}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T07:13:37,097 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731827616986Initializing all the Stores at 1731827616988 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731827616989 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827616990 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827616990Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827616990Cleaning up temporary data from old regions at 1731827617080 (+90 ms)Region opened successfully at 1731827617097 (+17 ms) 2024-11-17T07:13:37,099 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T07:13:37,135 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e779548, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1dd62e24b303/172.17.0.2:0 2024-11-17T07:13:37,175 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T07:13:37,187 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T07:13:37,187 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T07:13:37,190 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T07:13:37,191 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-17T07:13:37,196 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-17T07:13:37,196 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T07:13:37,222 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T07:13:37,231 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T07:13:37,234 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T07:13:37,236 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T07:13:37,237 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T07:13:37,239 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T07:13:37,241 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T07:13:37,244 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T07:13:37,246 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T07:13:37,247 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T07:13:37,250 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T07:13:37,269 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T07:13:37,271 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T07:13:37,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T07:13:37,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T07:13:37,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:13:37,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:13:37,278 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=1dd62e24b303,36491,1731827613877, sessionid=0x10124c627cf0000, setting cluster-up flag (Was=false) 2024-11-17T07:13:37,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:13:37,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:13:37,301 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T07:13:37,303 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1dd62e24b303,36491,1731827613877 2024-11-17T07:13:37,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:13:37,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:13:37,316 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T07:13:37,318 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=1dd62e24b303,36491,1731827613877 2024-11-17T07:13:37,324 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T07:13:37,399 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T07:13:37,405 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(746): ClusterId : d37c6a52-7167-4bf7-960c-03142c6fe8ff 2024-11-17T07:13:37,407 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T07:13:37,411 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T07:13:37,412 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T07:13:37,412 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T07:13:37,415 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T07:13:37,415 DEBUG [RS:0;1dd62e24b303:36419 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3eec19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=1dd62e24b303/172.17.0.2:0 2024-11-17T07:13:37,418 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T07:13:37,424 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 1dd62e24b303,36491,1731827613877 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T07:13:37,430 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;1dd62e24b303:36419 2024-11-17T07:13:37,432 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/1dd62e24b303:0, corePoolSize=5, maxPoolSize=5 2024-11-17T07:13:37,432 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/1dd62e24b303:0, corePoolSize=5, maxPoolSize=5 2024-11-17T07:13:37,432 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/1dd62e24b303:0, corePoolSize=5, maxPoolSize=5 2024-11-17T07:13:37,432 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/1dd62e24b303:0, corePoolSize=5, maxPoolSize=5 2024-11-17T07:13:37,432 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/1dd62e24b303:0, corePoolSize=10, maxPoolSize=10 2024-11-17T07:13:37,433 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/1dd62e24b303:0, corePoolSize=1, maxPoolSize=1 2024-11-17T07:13:37,433 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/1dd62e24b303:0, corePoolSize=2, maxPoolSize=2 2024-11-17T07:13:37,433 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/1dd62e24b303:0, corePoolSize=1, maxPoolSize=1 2024-11-17T07:13:37,433 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T07:13:37,433 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T07:13:37,433 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T07:13:37,435 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731827647435 2024-11-17T07:13:37,436 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(2659): reportForDuty to master=1dd62e24b303,36491,1731827613877 with port=36419, startcode=1731827614816 2024-11-17T07:13:37,437 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T07:13:37,438 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T07:13:37,438 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T07:13:37,439 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T07:13:37,442 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T07:13:37,442 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T07:13:37,442 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T07:13:37,443 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T07:13:37,443 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,445 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:37,445 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T07:13:37,447 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T07:13:37,449 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T07:13:37,449 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T07:13:37,451 DEBUG [RS:0;1dd62e24b303:36419 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T07:13:37,451 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T07:13:37,452 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T07:13:37,454 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/1dd62e24b303:0:becomeActiveMaster-HFileCleaner.large.0-1731827617453,5,FailOnTimeoutGroup] 2024-11-17T07:13:37,454 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/1dd62e24b303:0:becomeActiveMaster-HFileCleaner.small.0-1731827617454,5,FailOnTimeoutGroup] 2024-11-17T07:13:37,455 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,455 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T07:13:37,456 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741831_1007 (size=1321) 2024-11-17T07:13:37,457 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,459 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T07:13:37,460 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650 2024-11-17T07:13:37,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741832_1008 (size=32) 2024-11-17T07:13:37,483 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:13:37,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T07:13:37,488 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T07:13:37,489 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:37,489 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42051, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T07:13:37,489 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T07:13:37,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T07:13:37,493 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T07:13:37,493 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:37,494 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T07:13:37,494 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T07:13:37,496 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36491 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 1dd62e24b303,36419,1731827614816 2024-11-17T07:13:37,497 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T07:13:37,497 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:37,498 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T07:13:37,499 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T07:13:37,499 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36491 {}] master.ServerManager(517): Registering regionserver=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:37,502 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T07:13:37,502 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:37,503 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T07:13:37,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T07:13:37,505 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740 2024-11-17T07:13:37,506 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740 2024-11-17T07:13:37,510 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T07:13:37,510 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T07:13:37,511 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T07:13:37,515 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T07:13:37,517 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650 2024-11-17T07:13:37,517 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44301 2024-11-17T07:13:37,517 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T07:13:37,521 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T07:13:37,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T07:13:37,522 DEBUG [RS:0;1dd62e24b303:36419 {}] zookeeper.ZKUtil(111): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/1dd62e24b303,36419,1731827614816 2024-11-17T07:13:37,522 WARN [RS:0;1dd62e24b303:36419 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T07:13:37,522 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68904937, jitterRate=0.02676357328891754}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T07:13:37,523 INFO [RS:0;1dd62e24b303:36419 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T07:13:37,523 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/WALs/1dd62e24b303,36419,1731827614816 2024-11-17T07:13:37,525 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [1dd62e24b303,36419,1731827614816] 2024-11-17T07:13:37,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731827617483Initializing all the Stores at 1731827617485 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731827617485Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731827617486 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827617486Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731827617486Cleaning up temporary data from old regions at 1731827617510 (+24 ms)Region opened successfully at 1731827617526 (+16 ms) 2024-11-17T07:13:37,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T07:13:37,526 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T07:13:37,526 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T07:13:37,527 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T07:13:37,527 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T07:13:37,535 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T07:13:37,535 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731827617526Disabling compacts and flushes for region at 1731827617526Disabling writes for close at 1731827617527 (+1 ms)Writing region close event to WAL at 1731827617534 (+7 ms)Closed at 1731827617535 (+1 ms) 2024-11-17T07:13:37,540 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T07:13:37,540 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T07:13:37,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T07:13:37,554 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T07:13:37,560 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T07:13:37,564 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T07:13:37,573 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T07:13:37,581 INFO [RS:0;1dd62e24b303:36419 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T07:13:37,581 INFO [RS:0;1dd62e24b303:36419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,582 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T07:13:37,590 INFO [RS:0;1dd62e24b303:36419 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T07:13:37,592 INFO [RS:0;1dd62e24b303:36419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,592 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/1dd62e24b303:0, corePoolSize=1, maxPoolSize=1 2024-11-17T07:13:37,593 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/1dd62e24b303:0, corePoolSize=1, maxPoolSize=1 2024-11-17T07:13:37,593 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/1dd62e24b303:0, corePoolSize=1, maxPoolSize=1 2024-11-17T07:13:37,594 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/1dd62e24b303:0, corePoolSize=1, maxPoolSize=1 2024-11-17T07:13:37,594 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/1dd62e24b303:0, corePoolSize=1, maxPoolSize=1 2024-11-17T07:13:37,594 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/1dd62e24b303:0, corePoolSize=2, maxPoolSize=2 2024-11-17T07:13:37,594 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0, corePoolSize=1, maxPoolSize=1 2024-11-17T07:13:37,595 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/1dd62e24b303:0, corePoolSize=1, maxPoolSize=1 2024-11-17T07:13:37,595 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/1dd62e24b303:0, corePoolSize=1, maxPoolSize=1 2024-11-17T07:13:37,595 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/1dd62e24b303:0, corePoolSize=1, maxPoolSize=1 2024-11-17T07:13:37,595 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/1dd62e24b303:0, corePoolSize=1, maxPoolSize=1 2024-11-17T07:13:37,595 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/1dd62e24b303:0, corePoolSize=1, maxPoolSize=1 2024-11-17T07:13:37,595 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/1dd62e24b303:0, corePoolSize=3, maxPoolSize=3 2024-11-17T07:13:37,595 DEBUG [RS:0;1dd62e24b303:36419 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0, corePoolSize=3, maxPoolSize=3 2024-11-17T07:13:37,599 INFO [RS:0;1dd62e24b303:36419 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,599 INFO [RS:0;1dd62e24b303:36419 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,599 INFO [RS:0;1dd62e24b303:36419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,599 INFO [RS:0;1dd62e24b303:36419 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,599 INFO [RS:0;1dd62e24b303:36419 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,599 INFO [RS:0;1dd62e24b303:36419 {}] hbase.ChoreService(168): Chore ScheduledChore name=1dd62e24b303,36419,1731827614816-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T07:13:37,622 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T07:13:37,625 INFO [RS:0;1dd62e24b303:36419 {}] hbase.ChoreService(168): Chore ScheduledChore name=1dd62e24b303,36419,1731827614816-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,625 INFO [RS:0;1dd62e24b303:36419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,625 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.Replication(171): 1dd62e24b303,36419,1731827614816 started 2024-11-17T07:13:37,653 INFO [RS:0;1dd62e24b303:36419 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:37,654 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1482): Serving as 1dd62e24b303,36419,1731827614816, RpcServer on 1dd62e24b303/172.17.0.2:36419, sessionid=0x10124c627cf0001 2024-11-17T07:13:37,655 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T07:13:37,655 DEBUG [RS:0;1dd62e24b303:36419 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 1dd62e24b303,36419,1731827614816 2024-11-17T07:13:37,655 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1dd62e24b303,36419,1731827614816' 2024-11-17T07:13:37,655 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T07:13:37,656 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T07:13:37,658 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T07:13:37,658 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T07:13:37,658 DEBUG [RS:0;1dd62e24b303:36419 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 1dd62e24b303,36419,1731827614816 2024-11-17T07:13:37,658 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '1dd62e24b303,36419,1731827614816' 2024-11-17T07:13:37,658 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T07:13:37,659 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T07:13:37,660 DEBUG [RS:0;1dd62e24b303:36419 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T07:13:37,660 INFO [RS:0;1dd62e24b303:36419 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T07:13:37,660 INFO [RS:0;1dd62e24b303:36419 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T07:13:37,715 WARN [1dd62e24b303:36491 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-17T07:13:37,767 INFO [RS:0;1dd62e24b303:36419 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T07:13:37,771 INFO [RS:0;1dd62e24b303:36419 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1dd62e24b303%2C36419%2C1731827614816, suffix=, logDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/WALs/1dd62e24b303,36419,1731827614816, archiveDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/oldWALs, maxLogs=32 2024-11-17T07:13:37,789 DEBUG [RS:0;1dd62e24b303:36419 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/WALs/1dd62e24b303,36419,1731827614816/1dd62e24b303%2C36419%2C1731827614816.1731827617774, exclude list is [], retry=0 2024-11-17T07:13:37,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46705,DS-2e1b16d1-e222-4f01-af4c-e06070bab600,DISK] 2024-11-17T07:13:37,800 INFO [RS:0;1dd62e24b303:36419 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/WALs/1dd62e24b303,36419,1731827614816/1dd62e24b303%2C36419%2C1731827614816.1731827617774 2024-11-17T07:13:37,800 DEBUG [RS:0;1dd62e24b303:36419 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43811:43811)] 2024-11-17T07:13:37,968 DEBUG [1dd62e24b303:36491 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-17T07:13:37,983 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:37,991 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1dd62e24b303,36419,1731827614816, state=OPENING 2024-11-17T07:13:37,999 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T07:13:38,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:13:38,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:13:38,002 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T07:13:38,003 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T07:13:38,004 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T07:13:38,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:13:38,208 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T07:13:38,212 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39253, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T07:13:38,223 INFO [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T07:13:38,223 INFO [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T07:13:38,224 INFO [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T07:13:38,228 INFO [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=1dd62e24b303%2C36419%2C1731827614816.meta, suffix=.meta, logDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/WALs/1dd62e24b303,36419,1731827614816, archiveDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/oldWALs, maxLogs=32 2024-11-17T07:13:38,245 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/WALs/1dd62e24b303,36419,1731827614816/1dd62e24b303%2C36419%2C1731827614816.meta.1731827618230.meta, exclude list is [], retry=0 2024-11-17T07:13:38,249 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46705,DS-2e1b16d1-e222-4f01-af4c-e06070bab600,DISK] 2024-11-17T07:13:38,252 INFO [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/WALs/1dd62e24b303,36419,1731827614816/1dd62e24b303%2C36419%2C1731827614816.meta.1731827618230.meta 2024-11-17T07:13:38,252 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43811:43811)] 2024-11-17T07:13:38,253 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T07:13:38,254 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T07:13:38,257 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T07:13:38,262 INFO [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T07:13:38,266 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T07:13:38,267 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:13:38,267 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T07:13:38,267 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T07:13:38,271 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-17T07:13:38,272 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T07:13:38,272 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:38,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T07:13:38,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-17T07:13:38,275 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T07:13:38,275 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:38,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T07:13:38,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-17T07:13:38,277 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T07:13:38,277 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:38,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T07:13:38,278 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-17T07:13:38,279 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T07:13:38,279 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:38,280 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T07:13:38,280 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T07:13:38,282 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740 2024-11-17T07:13:38,284 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740 2024-11-17T07:13:38,286 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T07:13:38,286 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T07:13:38,287 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T07:13:38,289 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T07:13:38,291 INFO [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73015521, jitterRate=0.08801604807376862}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T07:13:38,291 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T07:13:38,293 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731827618268Writing region info on filesystem at 1731827618268Initializing all the Stores at 1731827618270 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731827618270Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731827618270Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827618270Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731827618270Cleaning up temporary data from old regions at 1731827618286 (+16 ms)Running coprocessor post-open hooks at 1731827618291 (+5 ms)Region opened successfully at 1731827618293 (+2 ms) 2024-11-17T07:13:38,300 INFO [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731827618199 2024-11-17T07:13:38,311 DEBUG [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T07:13:38,311 INFO [RS_OPEN_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T07:13:38,313 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:38,315 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 1dd62e24b303,36419,1731827614816, state=OPEN 2024-11-17T07:13:38,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T07:13:38,320 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T07:13:38,320 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T07:13:38,320 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T07:13:38,321 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:38,325 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T07:13:38,326 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=1dd62e24b303,36419,1731827614816 in 315 msec 2024-11-17T07:13:38,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T07:13:38,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 779 msec 2024-11-17T07:13:38,333 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T07:13:38,333 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T07:13:38,355 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:13:38,357 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:13:38,378 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:13:38,380 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35263, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:13:38,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0450 sec 2024-11-17T07:13:38,401 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731827618401, completionTime=-1 2024-11-17T07:13:38,403 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-17T07:13:38,403 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-17T07:13:38,430 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-17T07:13:38,430 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731827678430 2024-11-17T07:13:38,430 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731827738430 2024-11-17T07:13:38,430 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 26 msec 2024-11-17T07:13:38,432 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1dd62e24b303,36491,1731827613877-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:38,432 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1dd62e24b303,36491,1731827613877-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:38,432 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1dd62e24b303,36491,1731827613877-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:38,434 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-1dd62e24b303:36491, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:38,434 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:38,435 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:38,441 DEBUG [master/1dd62e24b303:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T07:13:38,462 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.513sec 2024-11-17T07:13:38,463 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T07:13:38,465 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T07:13:38,466 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T07:13:38,466 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T07:13:38,466 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T07:13:38,467 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1dd62e24b303,36491,1731827613877-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T07:13:38,467 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1dd62e24b303,36491,1731827613877-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T07:13:38,476 DEBUG [master/1dd62e24b303:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T07:13:38,477 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T07:13:38,477 INFO [master/1dd62e24b303:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=1dd62e24b303,36491,1731827613877-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T07:13:38,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f081a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:38,521 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T07:13:38,521 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T07:13:38,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:13:38,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:13:38,537 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:13:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:13:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:13:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@531238f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:38,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:13:38,546 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:13:38,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:13:38,549 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:13:38,553 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62702105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:38,553 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:13:38,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:13:38,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:13:38,564 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:13:38,588 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=1dd62e24b303,36491,1731827613877 2024-11-17T07:13:38,606 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=258, ProcessCount=11, AvailableMemoryMB=2049 2024-11-17T07:13:38,617 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T07:13:38,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 1dd62e24b303,36491,1731827613877 2024-11-17T07:13:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@680215db 2024-11-17T07:13:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T07:13:38,652 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T07:13:38,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36491 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T07:13:38,660 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36491 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T07:13:38,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T07:13:38,676 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T07:13:38,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36491 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-17T07:13:38,679 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:38,682 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T07:13:38,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T07:13:38,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741835_1011 (size=1061) 2024-11-17T07:13:38,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T07:13:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T07:13:39,104 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650 2024-11-17T07:13:39,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741836_1012 (size=53) 2024-11-17T07:13:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T07:13:39,521 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:13:39,521 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cccbc0d9c2f0bc3781663c8c9bb7223f, disabling compactions & flushes 2024-11-17T07:13:39,521 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:39,522 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:39,522 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. after waiting 0 ms 2024-11-17T07:13:39,522 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:39,522 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:39,522 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cccbc0d9c2f0bc3781663c8c9bb7223f: Waiting for close lock at 1731827619521Disabling compacts and flushes for region at 1731827619521Disabling writes for close at 1731827619522 (+1 ms)Writing region close event to WAL at 1731827619522Closed at 1731827619522 2024-11-17T07:13:39,524 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T07:13:39,530 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731827619525"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731827619525"}]},"ts":"1731827619525"} 2024-11-17T07:13:39,534 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T07:13:39,536 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T07:13:39,539 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827619536"}]},"ts":"1731827619536"} 2024-11-17T07:13:39,545 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T07:13:39,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cccbc0d9c2f0bc3781663c8c9bb7223f, ASSIGN}] 2024-11-17T07:13:39,550 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cccbc0d9c2f0bc3781663c8c9bb7223f, ASSIGN 2024-11-17T07:13:39,552 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cccbc0d9c2f0bc3781663c8c9bb7223f, ASSIGN; state=OFFLINE, location=1dd62e24b303,36419,1731827614816; forceNewPlan=false, retain=false 2024-11-17T07:13:39,704 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cccbc0d9c2f0bc3781663c8c9bb7223f, regionState=OPENING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:39,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cccbc0d9c2f0bc3781663c8c9bb7223f, ASSIGN because future has completed 2024-11-17T07:13:39,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:13:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T07:13:39,871 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:39,871 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} 2024-11-17T07:13:39,872 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:39,872 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:13:39,872 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:39,872 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:39,874 INFO [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:39,878 INFO [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:13:39,879 INFO [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cccbc0d9c2f0bc3781663c8c9bb7223f columnFamilyName A 2024-11-17T07:13:39,879 DEBUG [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:39,880 INFO [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] regionserver.HStore(327): Store=cccbc0d9c2f0bc3781663c8c9bb7223f/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:13:39,881 INFO [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:39,883 INFO [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:13:39,883 INFO [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cccbc0d9c2f0bc3781663c8c9bb7223f columnFamilyName B 2024-11-17T07:13:39,883 DEBUG [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:39,884 INFO [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] regionserver.HStore(327): Store=cccbc0d9c2f0bc3781663c8c9bb7223f/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:13:39,884 INFO [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:39,886 INFO [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:13:39,887 INFO [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cccbc0d9c2f0bc3781663c8c9bb7223f columnFamilyName C 2024-11-17T07:13:39,887 DEBUG [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:13:39,888 INFO [StoreOpener-cccbc0d9c2f0bc3781663c8c9bb7223f-1 {}] regionserver.HStore(327): Store=cccbc0d9c2f0bc3781663c8c9bb7223f/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:13:39,888 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:39,888 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:39,890 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:39,890 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:39,892 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:39,892 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:39,893 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T07:13:39,895 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:39,899 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T07:13:39,900 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cccbc0d9c2f0bc3781663c8c9bb7223f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68112256, jitterRate=0.014951705932617188}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T07:13:39,900 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:39,901 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cccbc0d9c2f0bc3781663c8c9bb7223f: Running coprocessor pre-open hook at 1731827619872Writing region info on filesystem at 1731827619872Initializing all the Stores at 1731827619874 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827619874Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827619874Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827619874Cleaning up temporary data from old regions at 1731827619892 (+18 ms)Running coprocessor post-open hooks at 1731827619900 (+8 ms)Region opened successfully at 1731827619901 (+1 ms) 2024-11-17T07:13:39,903 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., pid=6, masterSystemTime=1731827619863 2024-11-17T07:13:39,907 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:39,907 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:39,908 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cccbc0d9c2f0bc3781663c8c9bb7223f, regionState=OPEN, openSeqNum=2, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:39,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:13:39,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T07:13:39,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 in 205 msec 2024-11-17T07:13:39,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T07:13:39,923 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cccbc0d9c2f0bc3781663c8c9bb7223f, ASSIGN in 371 msec 2024-11-17T07:13:39,924 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T07:13:39,924 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827619924"}]},"ts":"1731827619924"} 2024-11-17T07:13:39,928 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T07:13:39,929 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T07:13:39,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2650 sec 2024-11-17T07:13:40,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T07:13:40,848 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:13:40,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,854 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:13:40,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:13:40,856 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:13:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:13:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:13:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5de9b018, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:13:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:13:40,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:13:40,860 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:13:40,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc499f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,862 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:13:40,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:13:40,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:13:40,866 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:13:40,868 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T07:13:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 1dd62e24b303,36491,1731827613877 2024-11-17T07:13:40,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@36f9a744 2024-11-17T07:13:40,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T07:13:40,876 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T07:13:40,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:13:40,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:13:40,883 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:13:40,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:13:40,883 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:13:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d22a118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:13:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:13:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:13:40,885 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:13:40,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,888 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:13:40,888 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:13:40,889 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:13:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:13:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:13:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5367b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:13:40,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:13:40,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:13:40,897 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:13:40,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39deefb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,900 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:13:40,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:13:40,902 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:13:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:13:40,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:13:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:13:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:13:40,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:13:40,906 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:13:40,908 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec6ad86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:13:40,909 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:13:40,911 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:13:40,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:13:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:13:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65ac3455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:13:40,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:13:40,913 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:13:40,915 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:13:40,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d699c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:13:40,921 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:13:40,922 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:13:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:13:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:13:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c2308e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:13:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:13:40,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:13:40,925 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:13:40,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee049f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,928 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,928 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:13:40,928 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:13:40,929 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:13:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:13:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:13:40,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60486904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:13:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:13:40,931 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:13:40,932 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:13:40,934 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1455c669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:13:40,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:13:40,939 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:13:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:13:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:13:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@614ff77d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:13:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:13:40,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:13:40,943 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:13:40,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c26f37b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:13:40,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:13:40,947 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:13:40,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:13:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:13:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fa1e09e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:13:40,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:13:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:13:40,950 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:13:40,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788043de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,952 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,952 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:13:40,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:13:40,953 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:13:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:13:40,953 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:13:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d5cf1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:13:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:13:40,954 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:13:40,955 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:13:40,956 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@caecd72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:13:40,960 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:13:40,960 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:13:40,961 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:13:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:13:40,963 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:13:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:13:40,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:13:40,964 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:13:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:13:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:13:40,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:13:40,965 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:13:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:13:40,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:13:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:13:40,966 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:13:40,967 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:13:40,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:13:40,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:13:40,968 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:13:40,969 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:13:40,969 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:13:40,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:13:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:13:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:13:40,970 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:13:40,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:13:40,971 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:13:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:13:40,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:13:40,972 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:13:40,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:13:40,974 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:13:40,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:13:40,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:13:40,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:13:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:13:40,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:13:40,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:13:40,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:13:40,977 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:13:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:13:40,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:13:40,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:13:40,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:13:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-17T07:13:40,990 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:13:40,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T07:13:40,992 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:13:40,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:13:40,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:13:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:41,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:13:41,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:41,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:41,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:41,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:41,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:41,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:41,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T07:13:41,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T07:13:41,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:41,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:41,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:41,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/5034bfd5dbf149c99152ec1d61f67794 is 50, key is test_row_0/A:col10/1731827621035/Put/seqid=0 2024-11-17T07:13:41,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827681195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,241 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827621139, max timestamp=1731827621203Num uniques 859; 2024-11-17T07:13:41,243 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:41,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827621139, max timestamp=1731827621203Num uniques 859; 2024-11-17T07:13:41,260 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:41,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741837_1013 (size=12001) 2024-11-17T07:13:41,277 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/5034bfd5dbf149c99152ec1d61f67794 2024-11-17T07:13:41,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827621139, max timestamp=1731827621203Num uniques 859; 2024-11-17T07:13:41,287 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:41,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T07:13:41,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T07:13:41,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:41,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:41,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:41,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827681363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827681366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827681366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827681369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,376 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,378 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/15f61ef21ae44a04bae58f481563367a is 50, key is test_row_0/B:col10/1731827621035/Put/seqid=0 2024-11-17T07:13:41,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741838_1014 (size=12001) 2024-11-17T07:13:41,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827681428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/15f61ef21ae44a04bae58f481563367a 2024-11-17T07:13:41,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827681489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,492 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827681491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827681493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827681494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,503 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/18216dfea1bd42aba8fb5d4f07f4d94f is 50, key is test_row_0/C:col10/1731827621035/Put/seqid=0 2024-11-17T07:13:41,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T07:13:41,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741839_1015 (size=12001) 2024-11-17T07:13:41,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:41,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:41,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:41,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T07:13:41,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827681649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827681698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827681709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827681709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T07:13:41,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:41,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:41,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:41,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827681718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T07:13:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:41,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:41,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/18216dfea1bd42aba8fb5d4f07f4d94f 2024-11-17T07:13:41,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/5034bfd5dbf149c99152ec1d61f67794 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5034bfd5dbf149c99152ec1d61f67794 2024-11-17T07:13:41,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:41,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827681967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:41,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:41,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5034bfd5dbf149c99152ec1d61f67794, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T07:13:41,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/15f61ef21ae44a04bae58f481563367a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/15f61ef21ae44a04bae58f481563367a 2024-11-17T07:13:42,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/15f61ef21ae44a04bae58f481563367a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T07:13:42,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/18216dfea1bd42aba8fb5d4f07f4d94f as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/18216dfea1bd42aba8fb5d4f07f4d94f 2024-11-17T07:13:42,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/18216dfea1bd42aba8fb5d4f07f4d94f, entries=150, sequenceid=13, filesize=11.7 K 2024-11-17T07:13:42,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T07:13:42,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:42,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for cccbc0d9c2f0bc3781663c8c9bb7223f in 968ms, sequenceid=13, compaction requested=false 2024-11-17T07:13:42,035 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T07:13:42,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:42,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:42,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-17T07:13:42,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:42,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:42,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:42,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:42,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:42,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:42,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/abb64e39ddf54b0b953da0718c77aed5 is 50, key is test_row_0/A:col10/1731827621356/Put/seqid=0 2024-11-17T07:13:42,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827682097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741840_1016 (size=14341) 2024-11-17T07:13:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T07:13:42,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827682107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827682109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827682113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T07:13:42,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:42,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827682218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827682248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827682300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,308 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827682300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,309 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T07:13:42,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827682478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827682478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,485 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827682498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T07:13:42,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:42,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827682518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827682519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,525 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/abb64e39ddf54b0b953da0718c77aed5 2024-11-17T07:13:42,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/7cacf3c77f6e42eabba31f3777cf264e is 50, key is test_row_0/B:col10/1731827621356/Put/seqid=0 2024-11-17T07:13:42,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741841_1017 (size=12001) 2024-11-17T07:13:42,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/7cacf3c77f6e42eabba31f3777cf264e 2024-11-17T07:13:42,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/880c637fd9b14b6cbd5abee6f89e053a is 50, key is test_row_0/C:col10/1731827621356/Put/seqid=0 2024-11-17T07:13:42,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741842_1018 (size=12001) 2024-11-17T07:13:42,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T07:13:42,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:42,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827682798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827682818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,819 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T07:13:42,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:42,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827682838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827682839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:42,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:42,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:42,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T07:13:42,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:42,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:42,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:43,050 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/880c637fd9b14b6cbd5abee6f89e053a 2024-11-17T07:13:43,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/abb64e39ddf54b0b953da0718c77aed5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/abb64e39ddf54b0b953da0718c77aed5 2024-11-17T07:13:43,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/abb64e39ddf54b0b953da0718c77aed5, entries=200, sequenceid=50, filesize=14.0 K 2024-11-17T07:13:43,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/7cacf3c77f6e42eabba31f3777cf264e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7cacf3c77f6e42eabba31f3777cf264e 2024-11-17T07:13:43,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7cacf3c77f6e42eabba31f3777cf264e, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T07:13:43,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/880c637fd9b14b6cbd5abee6f89e053a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/880c637fd9b14b6cbd5abee6f89e053a 2024-11-17T07:13:43,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/880c637fd9b14b6cbd5abee6f89e053a, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T07:13:43,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cccbc0d9c2f0bc3781663c8c9bb7223f in 1076ms, sequenceid=50, compaction requested=false 2024-11-17T07:13:43,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T07:13:43,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-17T07:13:43,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:43,139 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:13:43,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:43,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:43,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/f2b64651920b442c9249258210a7ffdc is 50, key is test_row_0/A:col10/1731827622090/Put/seqid=0 2024-11-17T07:13:43,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741843_1019 (size=12001) 2024-11-17T07:13:43,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:43,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:43,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827623378, max timestamp=1731827623404Num uniques 859; 2024-11-17T07:13:43,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:43,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827623378, max timestamp=1731827623403Num uniques 859; 2024-11-17T07:13:43,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:43,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827623378, max timestamp=1731827623404Num uniques 859; 2024-11-17T07:13:43,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:43,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731827623455, max timestamp=1731827623476Num uniques 1018; 2024-11-17T07:13:43,486 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T07:13:43,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731827623455, max timestamp=1731827623476Num uniques 1018; 2024-11-17T07:13:43,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731827623455, max timestamp=1731827623476Num uniques 1018; 2024-11-17T07:13:43,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T07:13:43,493 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T07:13:43,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731827623522, max timestamp=1731827623545Num uniques 1027; 2024-11-17T07:13:43,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-17T07:13:43,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731827623522, max timestamp=1731827623545Num uniques 1027; 2024-11-17T07:13:43,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-17T07:13:43,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827683529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731827623522, max timestamp=1731827623545Num uniques 1027; 2024-11-17T07:13:43,555 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-17T07:13:43,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,556 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,577 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/f2b64651920b442c9249258210a7ffdc 2024-11-17T07:13:43,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/a9cc6888f6724ab7981789b6e3211505 is 50, key is test_row_0/B:col10/1731827622090/Put/seqid=0 2024-11-17T07:13:43,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827683615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827683621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741844_1020 (size=12001) 2024-11-17T07:13:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827683623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827683624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,634 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,634 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/a9cc6888f6724ab7981789b6e3211505 2024-11-17T07:13:43,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d452c1c7307f4481a8cb08cbd4fc747a is 50, key is test_row_0/C:col10/1731827622090/Put/seqid=0 2024-11-17T07:13:43,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741845_1021 (size=12001) 2024-11-17T07:13:43,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,672 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d452c1c7307f4481a8cb08cbd4fc747a 2024-11-17T07:13:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827683668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/f2b64651920b442c9249258210a7ffdc as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f2b64651920b442c9249258210a7ffdc 2024-11-17T07:13:43,710 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f2b64651920b442c9249258210a7ffdc, entries=150, sequenceid=60, filesize=11.7 K 2024-11-17T07:13:43,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/a9cc6888f6724ab7981789b6e3211505 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/a9cc6888f6724ab7981789b6e3211505 2024-11-17T07:13:43,732 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/a9cc6888f6724ab7981789b6e3211505, entries=150, sequenceid=60, filesize=11.7 K 2024-11-17T07:13:43,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827683727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,735 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d452c1c7307f4481a8cb08cbd4fc747a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d452c1c7307f4481a8cb08cbd4fc747a 2024-11-17T07:13:43,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827683737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827683737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827683738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,754 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d452c1c7307f4481a8cb08cbd4fc747a, entries=150, sequenceid=60, filesize=11.7 K 2024-11-17T07:13:43,757 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cccbc0d9c2f0bc3781663c8c9bb7223f in 618ms, sequenceid=60, compaction requested=true 2024-11-17T07:13:43,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:43,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:43,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-17T07:13:43,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-17T07:13:43,768 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T07:13:43,769 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7700 sec 2024-11-17T07:13:43,772 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.7910 sec 2024-11-17T07:13:43,780 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T07:13:43,856 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T07:13:43,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:43,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=140.89 KB heapSize=393.84 KB 2024-11-17T07:13:43,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:43,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:43,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:43,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:43,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:43,883 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:43,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/7ec4ad4ac3be4f33a4e3f6ac60bd5a52 is 50, key is test_row_0/A:col10/1731827623579/Put/seqid=0 2024-11-17T07:13:43,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827683937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827683948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827683948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741846_1022 (size=12051) 2024-11-17T07:13:43,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827683958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:43,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827683959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:44,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827684058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:44,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:44,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:44,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827684259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,263 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:44,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827684259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:44,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:44,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827684268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827684268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:44,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827684277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:44,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,284 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:44,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/7ec4ad4ac3be4f33a4e3f6ac60bd5a52 2024-11-17T07:13:44,388 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T07:13:44,392 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-17T07:13:44,394 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T07:13:44,394 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-17T07:13:44,395 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T07:13:44,395 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-17T07:13:44,396 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T07:13:44,396 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T07:13:44,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/5b479f11f68e489b8fde70aa061dcda6 is 50, key is test_row_0/B:col10/1731827623579/Put/seqid=0 2024-11-17T07:13:44,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741847_1023 (size=12051) 2024-11-17T07:13:44,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/5b479f11f68e489b8fde70aa061dcda6 2024-11-17T07:13:44,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d9d9b493102344439feaed5995c38cbc is 50, key is test_row_0/C:col10/1731827623579/Put/seqid=0 2024-11-17T07:13:44,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741848_1024 (size=12051) 2024-11-17T07:13:44,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d9d9b493102344439feaed5995c38cbc 2024-11-17T07:13:44,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/7ec4ad4ac3be4f33a4e3f6ac60bd5a52 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7ec4ad4ac3be4f33a4e3f6ac60bd5a52 2024-11-17T07:13:44,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7ec4ad4ac3be4f33a4e3f6ac60bd5a52, entries=150, sequenceid=130, filesize=11.8 K 2024-11-17T07:13:44,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/5b479f11f68e489b8fde70aa061dcda6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/5b479f11f68e489b8fde70aa061dcda6 2024-11-17T07:13:44,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/5b479f11f68e489b8fde70aa061dcda6, entries=150, sequenceid=130, filesize=11.8 K 2024-11-17T07:13:44,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d9d9b493102344439feaed5995c38cbc as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d9d9b493102344439feaed5995c38cbc 2024-11-17T07:13:44,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d9d9b493102344439feaed5995c38cbc, entries=150, sequenceid=130, filesize=11.8 K 2024-11-17T07:13:44,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~393.80 KB/403248, currentSize=40.25 KB/41220 for cccbc0d9c2f0bc3781663c8c9bb7223f in 698ms, sequenceid=130, compaction requested=true 2024-11-17T07:13:44,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:44,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:44,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:13:44,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:44,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:13:44,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:44,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:13:44,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:44,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:13:44,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:44,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:44,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:44,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:44,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:44,588 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:44,590 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:13:44,590 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:13:44,596 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:13:44,596 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:13:44,598 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/B is initiating minor compaction (all files) 2024-11-17T07:13:44,599 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/B in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:44,599 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/15f61ef21ae44a04bae58f481563367a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7cacf3c77f6e42eabba31f3777cf264e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/a9cc6888f6724ab7981789b6e3211505, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/5b479f11f68e489b8fde70aa061dcda6] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=46.9 K 2024-11-17T07:13:44,600 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 15f61ef21ae44a04bae58f481563367a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731827621035 2024-11-17T07:13:44,601 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/A is initiating minor compaction (all files) 2024-11-17T07:13:44,601 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/A in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:44,601 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5034bfd5dbf149c99152ec1d61f67794, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/abb64e39ddf54b0b953da0718c77aed5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f2b64651920b442c9249258210a7ffdc, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7ec4ad4ac3be4f33a4e3f6ac60bd5a52] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=49.2 K 2024-11-17T07:13:44,605 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cacf3c77f6e42eabba31f3777cf264e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731827621355 2024-11-17T07:13:44,607 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a9cc6888f6724ab7981789b6e3211505, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731827622090 2024-11-17T07:13:44,608 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b479f11f68e489b8fde70aa061dcda6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731827623579 2024-11-17T07:13:44,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/a66e896b67e64a2bb67f50a92b57d1cc is 50, key is test_row_0/A:col10/1731827623896/Put/seqid=0 2024-11-17T07:13:44,609 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5034bfd5dbf149c99152ec1d61f67794, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731827621035 2024-11-17T07:13:44,614 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting abb64e39ddf54b0b953da0718c77aed5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731827621355 2024-11-17T07:13:44,617 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2b64651920b442c9249258210a7ffdc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731827622090 2024-11-17T07:13:44,622 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ec4ad4ac3be4f33a4e3f6ac60bd5a52, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731827623579 2024-11-17T07:13:44,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741849_1025 (size=14541) 2024-11-17T07:13:44,634 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/a66e896b67e64a2bb67f50a92b57d1cc 2024-11-17T07:13:44,679 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#A#compaction#13 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:44,680 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/603373856f174b35b023d3788736e1f3 is 50, key is test_row_0/A:col10/1731827623579/Put/seqid=0 2024-11-17T07:13:44,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/2d1195ccc6fd4c199355d06bb6fc0a76 is 50, key is test_row_0/B:col10/1731827623896/Put/seqid=0 2024-11-17T07:13:44,683 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#B#compaction#14 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:44,684 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/96fb862a18b44ccea403db0d0ef64542 is 50, key is test_row_0/B:col10/1731827623579/Put/seqid=0 2024-11-17T07:13:44,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741850_1026 (size=12189) 2024-11-17T07:13:44,740 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/603373856f174b35b023d3788736e1f3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/603373856f174b35b023d3788736e1f3 2024-11-17T07:13:44,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741851_1027 (size=12189) 2024-11-17T07:13:44,759 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/96fb862a18b44ccea403db0d0ef64542 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/96fb862a18b44ccea403db0d0ef64542 2024-11-17T07:13:44,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741852_1028 (size=12151) 2024-11-17T07:13:44,769 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/2d1195ccc6fd4c199355d06bb6fc0a76 2024-11-17T07:13:44,782 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/A of cccbc0d9c2f0bc3781663c8c9bb7223f into 603373856f174b35b023d3788736e1f3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:44,782 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:44,787 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/A, priority=12, startTime=1731827624583; duration=0sec 2024-11-17T07:13:44,787 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:44,788 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:A 2024-11-17T07:13:44,788 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:13:44,793 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/B of cccbc0d9c2f0bc3781663c8c9bb7223f into 96fb862a18b44ccea403db0d0ef64542(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:44,793 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:44,793 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/B, priority=12, startTime=1731827624587; duration=0sec 2024-11-17T07:13:44,793 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:44,793 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:B 2024-11-17T07:13:44,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d4dc5400ab0e4f3c94d6741de24632f8 is 50, key is test_row_0/C:col10/1731827623896/Put/seqid=0 2024-11-17T07:13:44,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731827624731, max timestamp=1731827624794Num uniques 859; 2024-11-17T07:13:44,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731827624731, max timestamp=1731827624794Num uniques 859; 2024-11-17T07:13:44,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:13:44,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:13:44,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827624731, max timestamp=1731827624794Num uniques 859; 2024-11-17T07:13:44,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:13:44,807 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48054 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:13:44,807 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/C is initiating minor compaction (all files) 2024-11-17T07:13:44,807 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/C in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:44,807 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/18216dfea1bd42aba8fb5d4f07f4d94f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/880c637fd9b14b6cbd5abee6f89e053a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d452c1c7307f4481a8cb08cbd4fc747a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d9d9b493102344439feaed5995c38cbc] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=46.9 K 2024-11-17T07:13:44,808 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18216dfea1bd42aba8fb5d4f07f4d94f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731827621035 2024-11-17T07:13:44,809 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 880c637fd9b14b6cbd5abee6f89e053a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731827621355 2024-11-17T07:13:44,810 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting d452c1c7307f4481a8cb08cbd4fc747a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731827622090 2024-11-17T07:13:44,811 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9d9b493102344439feaed5995c38cbc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731827623579 2024-11-17T07:13:44,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741853_1029 (size=12151) 2024-11-17T07:13:44,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d4dc5400ab0e4f3c94d6741de24632f8 2024-11-17T07:13:44,848 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#C#compaction#17 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:44,849 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/56191a8cf672443bb5a0cde616a69456 is 50, key is test_row_0/C:col10/1731827623579/Put/seqid=0 2024-11-17T07:13:44,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/a66e896b67e64a2bb67f50a92b57d1cc as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a66e896b67e64a2bb67f50a92b57d1cc 2024-11-17T07:13:44,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741854_1030 (size=12189) 2024-11-17T07:13:44,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a66e896b67e64a2bb67f50a92b57d1cc, entries=200, sequenceid=140, filesize=14.2 K 2024-11-17T07:13:44,880 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/56191a8cf672443bb5a0cde616a69456 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/56191a8cf672443bb5a0cde616a69456 2024-11-17T07:13:44,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/2d1195ccc6fd4c199355d06bb6fc0a76 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/2d1195ccc6fd4c199355d06bb6fc0a76 2024-11-17T07:13:44,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/2d1195ccc6fd4c199355d06bb6fc0a76, entries=150, sequenceid=140, filesize=11.9 K 2024-11-17T07:13:44,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d4dc5400ab0e4f3c94d6741de24632f8 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d4dc5400ab0e4f3c94d6741de24632f8 2024-11-17T07:13:44,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d4dc5400ab0e4f3c94d6741de24632f8, entries=150, sequenceid=140, filesize=11.9 K 2024-11-17T07:13:44,916 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/C of cccbc0d9c2f0bc3781663c8c9bb7223f into 56191a8cf672443bb5a0cde616a69456(size=11.9 K), total size for store is 23.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:44,916 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:44,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for cccbc0d9c2f0bc3781663c8c9bb7223f in 329ms, sequenceid=140, compaction requested=false 2024-11-17T07:13:44,916 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/C, priority=12, startTime=1731827624587; duration=0sec 2024-11-17T07:13:44,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:44,917 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:44,917 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:C 2024-11-17T07:13:44,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:44,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-17T07:13:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:44,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:44,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/bd1b53ad0e8b42d49ac7ff8beec281d3 is 50, key is test_row_0/A:col10/1731827624917/Put/seqid=0 2024-11-17T07:13:44,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:44,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:44,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827684944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:44,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827684944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:44,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:44,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827684947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:44,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:44,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827684953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:44,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827684954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:44,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:44,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741855_1031 (size=16931) 2024-11-17T07:13:44,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/bd1b53ad0e8b42d49ac7ff8beec281d3 2024-11-17T07:13:45,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/276634b39b144d8eb784ba8789701ba4 is 50, key is test_row_0/B:col10/1731827624917/Put/seqid=0 2024-11-17T07:13:45,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741856_1032 (size=12151) 2024-11-17T07:13:45,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/276634b39b144d8eb784ba8789701ba4 2024-11-17T07:13:45,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/9b975ca33ff742a08871a3935f657deb is 50, key is test_row_0/C:col10/1731827624917/Put/seqid=0 2024-11-17T07:13:45,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741857_1033 (size=12151) 2024-11-17T07:13:45,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/9b975ca33ff742a08871a3935f657deb 2024-11-17T07:13:45,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827685058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827685060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,069 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/bd1b53ad0e8b42d49ac7ff8beec281d3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/bd1b53ad0e8b42d49ac7ff8beec281d3 2024-11-17T07:13:45,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827685067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827685068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827685077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/bd1b53ad0e8b42d49ac7ff8beec281d3, entries=250, sequenceid=180, filesize=16.5 K 2024-11-17T07:13:45,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/276634b39b144d8eb784ba8789701ba4 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/276634b39b144d8eb784ba8789701ba4 2024-11-17T07:13:45,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/276634b39b144d8eb784ba8789701ba4, entries=150, sequenceid=180, filesize=11.9 K 2024-11-17T07:13:45,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/9b975ca33ff742a08871a3935f657deb as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9b975ca33ff742a08871a3935f657deb 2024-11-17T07:13:45,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9b975ca33ff742a08871a3935f657deb, entries=150, sequenceid=180, filesize=11.9 K 2024-11-17T07:13:45,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for cccbc0d9c2f0bc3781663c8c9bb7223f in 207ms, sequenceid=180, compaction requested=true 2024-11-17T07:13:45,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:45,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:13:45,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:45,125 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:45,125 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:45,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:13:45,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:45,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:13:45,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:45,130 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:45,130 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/A is initiating minor compaction (all files) 2024-11-17T07:13:45,130 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/A in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,130 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/603373856f174b35b023d3788736e1f3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a66e896b67e64a2bb67f50a92b57d1cc, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/bd1b53ad0e8b42d49ac7ff8beec281d3] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=42.6 K 2024-11-17T07:13:45,130 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36491 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:45,131 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/B is initiating minor compaction (all files) 2024-11-17T07:13:45,131 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/B in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,131 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/96fb862a18b44ccea403db0d0ef64542, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/2d1195ccc6fd4c199355d06bb6fc0a76, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/276634b39b144d8eb784ba8789701ba4] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=35.6 K 2024-11-17T07:13:45,131 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 603373856f174b35b023d3788736e1f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731827623579 2024-11-17T07:13:45,132 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 96fb862a18b44ccea403db0d0ef64542, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731827623579 2024-11-17T07:13:45,132 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting a66e896b67e64a2bb67f50a92b57d1cc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731827623896 2024-11-17T07:13:45,133 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d1195ccc6fd4c199355d06bb6fc0a76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731827623896 2024-11-17T07:13:45,134 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd1b53ad0e8b42d49ac7ff8beec281d3, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731827624819 2024-11-17T07:13:45,134 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 276634b39b144d8eb784ba8789701ba4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731827624862 2024-11-17T07:13:45,163 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#B#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:45,164 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/d5f3033be5d3427da198bbbf2c13b462 is 50, key is test_row_0/B:col10/1731827624917/Put/seqid=0 2024-11-17T07:13:45,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-17T07:13:45,167 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:13:45,168 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:13:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-17T07:13:45,171 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:45,172 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:13:45,172 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/d38357fbb7064a00a936e311bfa91ea8 is 50, key is test_row_0/A:col10/1731827624917/Put/seqid=0 2024-11-17T07:13:45,173 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:13:45,173 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:13:45,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T07:13:45,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741859_1035 (size=12391) 2024-11-17T07:13:45,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741858_1034 (size=12391) 2024-11-17T07:13:45,226 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/d5f3033be5d3427da198bbbf2c13b462 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d5f3033be5d3427da198bbbf2c13b462 2024-11-17T07:13:45,240 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/B of cccbc0d9c2f0bc3781663c8c9bb7223f into d5f3033be5d3427da198bbbf2c13b462(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:45,240 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:45,240 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/B, priority=13, startTime=1731827625125; duration=0sec 2024-11-17T07:13:45,240 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:45,240 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:B 2024-11-17T07:13:45,241 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:45,245 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36491 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:45,245 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/C is initiating minor compaction (all files) 2024-11-17T07:13:45,245 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/C in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,245 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/56191a8cf672443bb5a0cde616a69456, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d4dc5400ab0e4f3c94d6741de24632f8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9b975ca33ff742a08871a3935f657deb] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=35.6 K 2024-11-17T07:13:45,246 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 56191a8cf672443bb5a0cde616a69456, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731827623579 2024-11-17T07:13:45,247 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting d4dc5400ab0e4f3c94d6741de24632f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1731827623896 2024-11-17T07:13:45,248 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b975ca33ff742a08871a3935f657deb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731827624862 2024-11-17T07:13:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:45,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:13:45,272 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#C#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:45,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:45,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:45,272 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:45,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:45,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:45,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:45,273 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/1c51a8c20b1c48a4bc46a3120ab519a4 is 50, key is test_row_0/C:col10/1731827624917/Put/seqid=0 2024-11-17T07:13:45,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T07:13:45,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/21e399f9e561430685df1b5bd42ea8c3 is 50, key is test_row_0/A:col10/1731827624949/Put/seqid=0 2024-11-17T07:13:45,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741860_1036 (size=12391) 2024-11-17T07:13:45,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T07:13:45,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:45,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741861_1037 (size=14541) 2024-11-17T07:13:45,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/21e399f9e561430685df1b5bd42ea8c3 2024-11-17T07:13:45,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827625329, max timestamp=1731827625348Num uniques 859; 2024-11-17T07:13:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827685342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:13:45,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827625329, max timestamp=1731827625348Num uniques 859; 2024-11-17T07:13:45,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:13:45,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827625329, max timestamp=1731827625348Num uniques 859; 2024-11-17T07:13:45,355 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:13:45,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/142ff8a9b5d84f98937f3fa59d34cd53 is 50, key is test_row_0/B:col10/1731827624949/Put/seqid=0 2024-11-17T07:13:45,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741862_1038 (size=12151) 2024-11-17T07:13:45,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/142ff8a9b5d84f98937f3fa59d34cd53 2024-11-17T07:13:45,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827685384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827685386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827685389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827685389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d5fa87b02b6f4419b0f165f350f99d5c is 50, key is test_row_0/C:col10/1731827624949/Put/seqid=0 2024-11-17T07:13:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,401 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741863_1039 (size=12151) 2024-11-17T07:13:45,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d5fa87b02b6f4419b0f165f350f99d5c 2024-11-17T07:13:45,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/21e399f9e561430685df1b5bd42ea8c3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/21e399f9e561430685df1b5bd42ea8c3 2024-11-17T07:13:45,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/21e399f9e561430685df1b5bd42ea8c3, entries=200, sequenceid=192, filesize=14.2 K 2024-11-17T07:13:45,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/142ff8a9b5d84f98937f3fa59d34cd53 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/142ff8a9b5d84f98937f3fa59d34cd53 2024-11-17T07:13:45,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827685458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/142ff8a9b5d84f98937f3fa59d34cd53, entries=150, sequenceid=192, filesize=11.9 K 2024-11-17T07:13:45,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d5fa87b02b6f4419b0f165f350f99d5c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d5fa87b02b6f4419b0f165f350f99d5c 2024-11-17T07:13:45,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T07:13:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d5fa87b02b6f4419b0f165f350f99d5c, entries=150, sequenceid=192, filesize=11.9 K 2024-11-17T07:13:45,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cccbc0d9c2f0bc3781663c8c9bb7223f in 223ms, sequenceid=192, compaction requested=false 2024-11-17T07:13:45,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:45,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T07:13:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:45,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-17T07:13:45,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:45,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:45,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:45,512 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:45,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:45,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:45,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/d5ee7059807b453cb8410eb1808e6510 is 50, key is test_row_0/A:col10/1731827625508/Put/seqid=0 2024-11-17T07:13:45,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827685523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827685525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827685528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827685531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741864_1040 (size=14541) 2024-11-17T07:13:45,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/d5ee7059807b453cb8410eb1808e6510 2024-11-17T07:13:45,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/71a33c9f65b94a52920ee5dacfa1761e is 50, key is test_row_0/B:col10/1731827625508/Put/seqid=0 2024-11-17T07:13:45,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741865_1041 (size=12151) 2024-11-17T07:13:45,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/71a33c9f65b94a52920ee5dacfa1761e 2024-11-17T07:13:45,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/29d25cc0ac88485bb0bcc01b22ae1a74 is 50, key is test_row_0/C:col10/1731827625508/Put/seqid=0 2024-11-17T07:13:45,616 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/d38357fbb7064a00a936e311bfa91ea8 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d38357fbb7064a00a936e311bfa91ea8 2024-11-17T07:13:45,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741866_1042 (size=12151) 2024-11-17T07:13:45,624 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/29d25cc0ac88485bb0bcc01b22ae1a74 2024-11-17T07:13:45,630 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/A of cccbc0d9c2f0bc3781663c8c9bb7223f into d38357fbb7064a00a936e311bfa91ea8(size=12.1 K), total size for store is 26.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:45,630 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:45,630 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/A, priority=13, startTime=1731827625125; duration=0sec 2024-11-17T07:13:45,631 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:45,631 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:A 2024-11-17T07:13:45,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/d5ee7059807b453cb8410eb1808e6510 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d5ee7059807b453cb8410eb1808e6510 2024-11-17T07:13:45,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T07:13:45,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:45,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d5ee7059807b453cb8410eb1808e6510, entries=200, sequenceid=227, filesize=14.2 K 2024-11-17T07:13:45,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/71a33c9f65b94a52920ee5dacfa1761e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/71a33c9f65b94a52920ee5dacfa1761e 2024-11-17T07:13:45,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827685647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827685647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827685648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827685648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/71a33c9f65b94a52920ee5dacfa1761e, entries=150, sequenceid=227, filesize=11.9 K 2024-11-17T07:13:45,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/29d25cc0ac88485bb0bcc01b22ae1a74 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/29d25cc0ac88485bb0bcc01b22ae1a74 2024-11-17T07:13:45,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/29d25cc0ac88485bb0bcc01b22ae1a74, entries=150, sequenceid=227, filesize=11.9 K 2024-11-17T07:13:45,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for cccbc0d9c2f0bc3781663c8c9bb7223f in 159ms, sequenceid=227, compaction requested=true 2024-11-17T07:13:45,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:45,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:A, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:13:45,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:45,671 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:45,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:B, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:13:45,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:45,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:13:45,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T07:13:45,672 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41473 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:45,673 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/A is initiating minor compaction (all files) 2024-11-17T07:13:45,673 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/A in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,673 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d38357fbb7064a00a936e311bfa91ea8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/21e399f9e561430685df1b5bd42ea8c3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d5ee7059807b453cb8410eb1808e6510] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=40.5 K 2024-11-17T07:13:45,673 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting d38357fbb7064a00a936e311bfa91ea8, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731827624862 2024-11-17T07:13:45,674 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e399f9e561430685df1b5bd42ea8c3, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731827624940 2024-11-17T07:13:45,675 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5ee7059807b453cb8410eb1808e6510, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731827625383 2024-11-17T07:13:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:45,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:13:45,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:45,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:45,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:45,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:45,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:45,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:45,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/20e47e2a01ad4cd6b2ac6056bb92db70 is 50, key is test_row_0/A:col10/1731827625520/Put/seqid=0 2024-11-17T07:13:45,697 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#A#compaction#31 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:45,698 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/5fa82aeb03db421db486e8c1c8e564d7 is 50, key is test_row_0/A:col10/1731827625508/Put/seqid=0 2024-11-17T07:13:45,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741867_1043 (size=12147) 2024-11-17T07:13:45,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741868_1044 (size=12493) 2024-11-17T07:13:45,745 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/1c51a8c20b1c48a4bc46a3120ab519a4 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/1c51a8c20b1c48a4bc46a3120ab519a4 2024-11-17T07:13:45,747 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/5fa82aeb03db421db486e8c1c8e564d7 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5fa82aeb03db421db486e8c1c8e564d7 2024-11-17T07:13:45,758 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/C of cccbc0d9c2f0bc3781663c8c9bb7223f into 1c51a8c20b1c48a4bc46a3120ab519a4(size=12.1 K), total size for store is 35.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:45,759 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:45,759 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/C, priority=13, startTime=1731827625128; duration=0sec 2024-11-17T07:13:45,759 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T07:13:45,759 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:C 2024-11-17T07:13:45,759 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:C 2024-11-17T07:13:45,759 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:45,761 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:45,761 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/B is initiating minor compaction (all files) 2024-11-17T07:13:45,761 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/B in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,762 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d5f3033be5d3427da198bbbf2c13b462, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/142ff8a9b5d84f98937f3fa59d34cd53, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/71a33c9f65b94a52920ee5dacfa1761e] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=35.8 K 2024-11-17T07:13:45,762 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting d5f3033be5d3427da198bbbf2c13b462, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731827624862 2024-11-17T07:13:45,762 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/A of cccbc0d9c2f0bc3781663c8c9bb7223f into 5fa82aeb03db421db486e8c1c8e564d7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:45,762 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:45,763 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/A, priority=13, startTime=1731827625671; duration=0sec 2024-11-17T07:13:45,763 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:45,763 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:A 2024-11-17T07:13:45,763 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 142ff8a9b5d84f98937f3fa59d34cd53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731827624948 2024-11-17T07:13:45,763 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:45,764 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 71a33c9f65b94a52920ee5dacfa1761e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731827625383 2024-11-17T07:13:45,766 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:45,766 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/C is initiating minor compaction (all files) 2024-11-17T07:13:45,767 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/C in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,768 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/1c51a8c20b1c48a4bc46a3120ab519a4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d5fa87b02b6f4419b0f165f350f99d5c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/29d25cc0ac88485bb0bcc01b22ae1a74] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=35.8 K 2024-11-17T07:13:45,769 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c51a8c20b1c48a4bc46a3120ab519a4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731827624862 2024-11-17T07:13:45,770 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5fa87b02b6f4419b0f165f350f99d5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731827624948 2024-11-17T07:13:45,771 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29d25cc0ac88485bb0bcc01b22ae1a74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731827625383 2024-11-17T07:13:45,780 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#B#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:45,781 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/9cf63011940643e7919c9b67b08edfcd is 50, key is test_row_0/B:col10/1731827625508/Put/seqid=0 2024-11-17T07:13:45,789 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:45,790 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d65104a3164947d7ad8ee361880ac158 is 50, key is test_row_0/C:col10/1731827625508/Put/seqid=0 2024-11-17T07:13:45,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T07:13:45,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:45,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T07:13:45,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741869_1045 (size=12493) 2024-11-17T07:13:45,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827625795, max timestamp=1731827625823Num uniques 859; 2024-11-17T07:13:45,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827625795, max timestamp=1731827625823Num uniques 859; 2024-11-17T07:13:45,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:45,828 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:45,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827625795, max timestamp=1731827625823Num uniques 859; 2024-11-17T07:13:45,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:45,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741870_1046 (size=12493) 2024-11-17T07:13:45,842 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/d65104a3164947d7ad8ee361880ac158 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d65104a3164947d7ad8ee361880ac158 2024-11-17T07:13:45,854 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/C of cccbc0d9c2f0bc3781663c8c9bb7223f into d65104a3164947d7ad8ee361880ac158(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:45,854 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:45,854 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/C, priority=13, startTime=1731827625671; duration=0sec 2024-11-17T07:13:45,854 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:45,854 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:C 2024-11-17T07:13:45,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827685877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827685889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827685890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827685891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827685892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T07:13:45,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:45,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:45,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:45,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827685997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:45,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:45,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827685998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827685998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827686007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827686007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T07:13:46,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:46,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:46,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:46,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:46,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:46,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:46,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/20e47e2a01ad4cd6b2ac6056bb92db70 2024-11-17T07:13:46,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/53aefa13f7b0494b96e100867d1aebe8 is 50, key is test_row_0/B:col10/1731827625520/Put/seqid=0 2024-11-17T07:13:46,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741871_1047 (size=9757) 2024-11-17T07:13:46,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/53aefa13f7b0494b96e100867d1aebe8 2024-11-17T07:13:46,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/7b4bcbc3b1404684bd9daaa5c2cc1e2c is 50, key is test_row_0/C:col10/1731827625520/Put/seqid=0 2024-11-17T07:13:46,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741872_1048 (size=9757) 2024-11-17T07:13:46,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/7b4bcbc3b1404684bd9daaa5c2cc1e2c 2024-11-17T07:13:46,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827686208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827686207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,211 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827686208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/20e47e2a01ad4cd6b2ac6056bb92db70 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/20e47e2a01ad4cd6b2ac6056bb92db70 2024-11-17T07:13:46,223 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/9cf63011940643e7919c9b67b08edfcd as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/9cf63011940643e7919c9b67b08edfcd 2024-11-17T07:13:46,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827686218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827686218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/20e47e2a01ad4cd6b2ac6056bb92db70, entries=150, sequenceid=239, filesize=11.9 K 2024-11-17T07:13:46,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/53aefa13f7b0494b96e100867d1aebe8 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/53aefa13f7b0494b96e100867d1aebe8 2024-11-17T07:13:46,236 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/B of cccbc0d9c2f0bc3781663c8c9bb7223f into 9cf63011940643e7919c9b67b08edfcd(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:46,237 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:46,237 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/B, priority=13, startTime=1731827625671; duration=0sec 2024-11-17T07:13:46,237 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:46,237 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:B 2024-11-17T07:13:46,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/53aefa13f7b0494b96e100867d1aebe8, entries=100, sequenceid=239, filesize=9.5 K 2024-11-17T07:13:46,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/7b4bcbc3b1404684bd9daaa5c2cc1e2c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/7b4bcbc3b1404684bd9daaa5c2cc1e2c 2024-11-17T07:13:46,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/7b4bcbc3b1404684bd9daaa5c2cc1e2c, entries=100, sequenceid=239, filesize=9.5 K 2024-11-17T07:13:46,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for cccbc0d9c2f0bc3781663c8c9bb7223f in 574ms, sequenceid=239, compaction requested=false 2024-11-17T07:13:46,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:46,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-17T07:13:46,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:46,264 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-17T07:13:46,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:46,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:46,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:46,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:46,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:46,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:46,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/4eb6f4b9787b45f4bcd78e29022e26a4 is 50, key is test_row_0/A:col10/1731827625884/Put/seqid=0 2024-11-17T07:13:46,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741873_1049 (size=12301) 2024-11-17T07:13:46,298 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/4eb6f4b9787b45f4bcd78e29022e26a4 2024-11-17T07:13:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T07:13:46,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/8ce49fb80da2445d9fbf3b5a31b48bc7 is 50, key is test_row_0/B:col10/1731827625884/Put/seqid=0 2024-11-17T07:13:46,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741874_1050 (size=12301) 2024-11-17T07:13:46,347 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/8ce49fb80da2445d9fbf3b5a31b48bc7 2024-11-17T07:13:46,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/2eea551a2db2492c931e0a1c5a947885 is 50, key is test_row_0/C:col10/1731827625884/Put/seqid=0 2024-11-17T07:13:46,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741875_1051 (size=12301) 2024-11-17T07:13:46,411 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/2eea551a2db2492c931e0a1c5a947885 2024-11-17T07:13:46,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/4eb6f4b9787b45f4bcd78e29022e26a4 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/4eb6f4b9787b45f4bcd78e29022e26a4 2024-11-17T07:13:46,441 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/4eb6f4b9787b45f4bcd78e29022e26a4, entries=150, sequenceid=280, filesize=12.0 K 2024-11-17T07:13:46,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/8ce49fb80da2445d9fbf3b5a31b48bc7 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/8ce49fb80da2445d9fbf3b5a31b48bc7 2024-11-17T07:13:46,462 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/8ce49fb80da2445d9fbf3b5a31b48bc7, entries=150, sequenceid=280, filesize=12.0 K 2024-11-17T07:13:46,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/2eea551a2db2492c931e0a1c5a947885 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2eea551a2db2492c931e0a1c5a947885 2024-11-17T07:13:46,475 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2eea551a2db2492c931e0a1c5a947885, entries=150, sequenceid=280, filesize=12.0 K 2024-11-17T07:13:46,478 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for cccbc0d9c2f0bc3781663c8c9bb7223f in 213ms, sequenceid=280, compaction requested=true 2024-11-17T07:13:46,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:46,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:46,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-17T07:13:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-17T07:13:46,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-17T07:13:46,484 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3070 sec 2024-11-17T07:13:46,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.3170 sec 2024-11-17T07:13:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:46,537 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:13:46,537 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:46,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:46,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:46,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:46,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:46,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:46,547 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/020e3eb62169417c82e4e1e375e4345b is 50, key is test_row_0/A:col10/1731827626524/Put/seqid=0 2024-11-17T07:13:46,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741876_1052 (size=14741) 2024-11-17T07:13:46,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/020e3eb62169417c82e4e1e375e4345b 2024-11-17T07:13:46,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/28cc6a096e8f4264b4a226a1e32bbd0a is 50, key is test_row_0/B:col10/1731827626524/Put/seqid=0 2024-11-17T07:13:46,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827686580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827686582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827626559, max timestamp=1731827626585Num uniques 859; 2024-11-17T07:13:46,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:13:46,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827626559, max timestamp=1731827626585Num uniques 859; 2024-11-17T07:13:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:13:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827626559, max timestamp=1731827626585Num uniques 859; 2024-11-17T07:13:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:13:46,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741877_1053 (size=12301) 2024-11-17T07:13:46,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/28cc6a096e8f4264b4a226a1e32bbd0a 2024-11-17T07:13:46,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/9a33e72adc3f4ea191099ce704cf8418 is 50, key is test_row_0/C:col10/1731827626524/Put/seqid=0 2024-11-17T07:13:46,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741878_1054 (size=12301) 2024-11-17T07:13:46,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827686635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827686635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827686638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827686697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827686697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827686757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827686758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827686758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827686907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827686917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827686977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827686977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827686977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/9a33e72adc3f4ea191099ce704cf8418 2024-11-17T07:13:47,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/020e3eb62169417c82e4e1e375e4345b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/020e3eb62169417c82e4e1e375e4345b 2024-11-17T07:13:47,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/020e3eb62169417c82e4e1e375e4345b, entries=200, sequenceid=291, filesize=14.4 K 2024-11-17T07:13:47,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/28cc6a096e8f4264b4a226a1e32bbd0a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/28cc6a096e8f4264b4a226a1e32bbd0a 2024-11-17T07:13:47,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/28cc6a096e8f4264b4a226a1e32bbd0a, entries=150, sequenceid=291, filesize=12.0 K 2024-11-17T07:13:47,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/9a33e72adc3f4ea191099ce704cf8418 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9a33e72adc3f4ea191099ce704cf8418 2024-11-17T07:13:47,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9a33e72adc3f4ea191099ce704cf8418, entries=150, sequenceid=291, filesize=12.0 K 2024-11-17T07:13:47,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cccbc0d9c2f0bc3781663c8c9bb7223f in 533ms, sequenceid=291, compaction requested=true 2024-11-17T07:13:47,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:47,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:13:47,071 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:13:47,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:47,071 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:13:47,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:13:47,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:47,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:13:47,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:47,074 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:13:47,074 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/B is initiating minor compaction (all files) 2024-11-17T07:13:47,074 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/B in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:47,074 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/9cf63011940643e7919c9b67b08edfcd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/53aefa13f7b0494b96e100867d1aebe8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/8ce49fb80da2445d9fbf3b5a31b48bc7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/28cc6a096e8f4264b4a226a1e32bbd0a] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=45.8 K 2024-11-17T07:13:47,074 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:13:47,075 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/A is initiating minor compaction (all files) 2024-11-17T07:13:47,075 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/A in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:47,075 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5fa82aeb03db421db486e8c1c8e564d7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/20e47e2a01ad4cd6b2ac6056bb92db70, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/4eb6f4b9787b45f4bcd78e29022e26a4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/020e3eb62169417c82e4e1e375e4345b] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=50.5 K 2024-11-17T07:13:47,075 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fa82aeb03db421db486e8c1c8e564d7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731827625383 2024-11-17T07:13:47,075 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cf63011940643e7919c9b67b08edfcd, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731827625383 2024-11-17T07:13:47,076 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20e47e2a01ad4cd6b2ac6056bb92db70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731827625520 2024-11-17T07:13:47,076 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 53aefa13f7b0494b96e100867d1aebe8, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731827625520 2024-11-17T07:13:47,077 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eb6f4b9787b45f4bcd78e29022e26a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731827625884 2024-11-17T07:13:47,077 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ce49fb80da2445d9fbf3b5a31b48bc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731827625884 2024-11-17T07:13:47,077 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 020e3eb62169417c82e4e1e375e4345b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731827626524 2024-11-17T07:13:47,077 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 28cc6a096e8f4264b4a226a1e32bbd0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731827626524 2024-11-17T07:13:47,095 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:47,096 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/4103d81e897840dfa681ddaa5771bd47 is 50, key is test_row_0/B:col10/1731827626524/Put/seqid=0 2024-11-17T07:13:47,103 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:47,104 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/b833a96f35aa452ca57d3fc76a292f04 is 50, key is test_row_0/A:col10/1731827626524/Put/seqid=0 2024-11-17T07:13:47,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741879_1055 (size=12779) 2024-11-17T07:13:47,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741880_1056 (size=12779) 2024-11-17T07:13:47,131 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/4103d81e897840dfa681ddaa5771bd47 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/4103d81e897840dfa681ddaa5771bd47 2024-11-17T07:13:47,131 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/b833a96f35aa452ca57d3fc76a292f04 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/b833a96f35aa452ca57d3fc76a292f04 2024-11-17T07:13:47,147 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/B of cccbc0d9c2f0bc3781663c8c9bb7223f into 4103d81e897840dfa681ddaa5771bd47(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:47,147 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:47,147 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/B, priority=12, startTime=1731827627071; duration=0sec 2024-11-17T07:13:47,147 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:47,147 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:B 2024-11-17T07:13:47,147 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:13:47,148 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/A of cccbc0d9c2f0bc3781663c8c9bb7223f into b833a96f35aa452ca57d3fc76a292f04(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:47,148 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:47,148 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/A, priority=12, startTime=1731827627071; duration=0sec 2024-11-17T07:13:47,148 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:47,148 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:A 2024-11-17T07:13:47,150 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:13:47,150 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/C is initiating minor compaction (all files) 2024-11-17T07:13:47,150 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/C in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:47,152 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d65104a3164947d7ad8ee361880ac158, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/7b4bcbc3b1404684bd9daaa5c2cc1e2c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2eea551a2db2492c931e0a1c5a947885, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9a33e72adc3f4ea191099ce704cf8418] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=45.8 K 2024-11-17T07:13:47,153 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting d65104a3164947d7ad8ee361880ac158, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731827625383 2024-11-17T07:13:47,153 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b4bcbc3b1404684bd9daaa5c2cc1e2c, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731827625520 2024-11-17T07:13:47,154 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eea551a2db2492c931e0a1c5a947885, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731827625884 2024-11-17T07:13:47,154 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a33e72adc3f4ea191099ce704cf8418, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731827626524 2024-11-17T07:13:47,175 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:47,176 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/14b39cb6dba6491a96ff132b7847af0d is 50, key is test_row_0/C:col10/1731827626524/Put/seqid=0 2024-11-17T07:13:47,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741881_1057 (size=12779) 2024-11-17T07:13:47,198 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/14b39cb6dba6491a96ff132b7847af0d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/14b39cb6dba6491a96ff132b7847af0d 2024-11-17T07:13:47,213 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/C of cccbc0d9c2f0bc3781663c8c9bb7223f into 14b39cb6dba6491a96ff132b7847af0d(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:47,213 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:47,213 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/C, priority=12, startTime=1731827627071; duration=0sec 2024-11-17T07:13:47,213 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:47,213 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:C 2024-11-17T07:13:47,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:47,222 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-17T07:13:47,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:47,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:47,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:47,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:47,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:47,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:47,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/c1bd9ff043e34fd09df31240e3c92b85 is 50, key is test_row_0/A:col10/1731827626632/Put/seqid=0 2024-11-17T07:13:47,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741882_1058 (size=14741) 2024-11-17T07:13:47,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/c1bd9ff043e34fd09df31240e3c92b85 2024-11-17T07:13:47,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827687257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827687260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/f6f2771ced354b88bc050db42db67e52 is 50, key is test_row_0/B:col10/1731827626632/Put/seqid=0 2024-11-17T07:13:47,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741883_1059 (size=12301) 2024-11-17T07:13:47,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827687288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827687288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827687297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-17T07:13:47,328 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:13:47,329 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:13:47,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-17T07:13:47,333 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:13:47,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T07:13:47,334 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:13:47,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:13:47,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827687367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827687377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T07:13:47,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T07:13:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:47,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:47,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:47,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:47,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827687577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827687587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T07:13:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T07:13:47,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:47,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:47,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:47,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:47,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:47,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:47,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/f6f2771ced354b88bc050db42db67e52 2024-11-17T07:13:47,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/78ebaeddc73b4a03986ed067088a1561 is 50, key is test_row_0/C:col10/1731827626632/Put/seqid=0 2024-11-17T07:13:47,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741884_1060 (size=12301) 2024-11-17T07:13:47,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/78ebaeddc73b4a03986ed067088a1561 2024-11-17T07:13:47,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/c1bd9ff043e34fd09df31240e3c92b85 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/c1bd9ff043e34fd09df31240e3c92b85 2024-11-17T07:13:47,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/c1bd9ff043e34fd09df31240e3c92b85, entries=200, sequenceid=329, filesize=14.4 K 2024-11-17T07:13:47,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/f6f2771ced354b88bc050db42db67e52 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/f6f2771ced354b88bc050db42db67e52 2024-11-17T07:13:47,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/f6f2771ced354b88bc050db42db67e52, entries=150, sequenceid=329, filesize=12.0 K 2024-11-17T07:13:47,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/78ebaeddc73b4a03986ed067088a1561 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/78ebaeddc73b4a03986ed067088a1561 2024-11-17T07:13:47,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/78ebaeddc73b4a03986ed067088a1561, entries=150, sequenceid=329, filesize=12.0 K 2024-11-17T07:13:47,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for cccbc0d9c2f0bc3781663c8c9bb7223f in 548ms, sequenceid=329, compaction requested=false 2024-11-17T07:13:47,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:47,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-17T07:13:47,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:47,803 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:13:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:47,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/a6e832f36f494a29bbc121901e3e8c12 is 50, key is test_row_0/A:col10/1731827627240/Put/seqid=0 2024-11-17T07:13:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:47,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:47,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741885_1061 (size=12301) 2024-11-17T07:13:47,835 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/a6e832f36f494a29bbc121901e3e8c12 2024-11-17T07:13:47,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/49856b44e4fe4abab7e1b205f0604798 is 50, key is test_row_0/B:col10/1731827627240/Put/seqid=0 2024-11-17T07:13:47,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741886_1062 (size=12301) 2024-11-17T07:13:47,856 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/49856b44e4fe4abab7e1b205f0604798 2024-11-17T07:13:47,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827627846, max timestamp=1731827627857Num uniques 859; 2024-11-17T07:13:47,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827627846, max timestamp=1731827627857Num uniques 859; 2024-11-17T07:13:47,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:13:47,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:13:47,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827627846, max timestamp=1731827627857Num uniques 859; 2024-11-17T07:13:47,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:13:47,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/0c39f90a90c2437cb71f56dfa587b671 is 50, key is test_row_0/C:col10/1731827627240/Put/seqid=0 2024-11-17T07:13:47,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741887_1063 (size=12301) 2024-11-17T07:13:47,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827687898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827687906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827687909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827687911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827687912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:47,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:47,920 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:47,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T07:13:48,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827688017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827688017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827688028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,030 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827688028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827688227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827688227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827688237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827688238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,292 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/0c39f90a90c2437cb71f56dfa587b671 2024-11-17T07:13:48,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/a6e832f36f494a29bbc121901e3e8c12 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a6e832f36f494a29bbc121901e3e8c12 2024-11-17T07:13:48,308 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a6e832f36f494a29bbc121901e3e8c12, entries=150, sequenceid=340, filesize=12.0 K 2024-11-17T07:13:48,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/49856b44e4fe4abab7e1b205f0604798 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/49856b44e4fe4abab7e1b205f0604798 2024-11-17T07:13:48,315 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/49856b44e4fe4abab7e1b205f0604798, entries=150, sequenceid=340, filesize=12.0 K 2024-11-17T07:13:48,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/0c39f90a90c2437cb71f56dfa587b671 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/0c39f90a90c2437cb71f56dfa587b671 2024-11-17T07:13:48,323 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/0c39f90a90c2437cb71f56dfa587b671, entries=150, sequenceid=340, filesize=12.0 K 2024-11-17T07:13:48,329 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for cccbc0d9c2f0bc3781663c8c9bb7223f in 527ms, sequenceid=340, compaction requested=true 2024-11-17T07:13:48,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:48,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:48,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-17T07:13:48,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-17T07:13:48,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-17T07:13:48,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 998 msec 2024-11-17T07:13:48,339 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.0070 sec 2024-11-17T07:13:48,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:48,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-17T07:13:48,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:48,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:48,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:48,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:48,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:48,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:48,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/baaf67a34abf4558b4001fb4ccfa29f3 is 50, key is test_row_0/A:col10/1731827627908/Put/seqid=0 2024-11-17T07:13:48,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741888_1064 (size=12301) 2024-11-17T07:13:48,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827688450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-17T07:13:48,467 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:13:48,468 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:13:48,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-17T07:13:48,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T07:13:48,470 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:13:48,471 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:13:48,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:13:48,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827688538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827688538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827688547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827688548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827688557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T07:13:48,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T07:13:48,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:48,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:48,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:48,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:48,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:48,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:48,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827688768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:48,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:48,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T07:13:48,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:48,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:48,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:48,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:48,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T07:13:48,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/baaf67a34abf4558b4001fb4ccfa29f3 2024-11-17T07:13:48,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/88a065ba61404bc18250c3f939fe4bf3 is 50, key is test_row_0/B:col10/1731827627908/Put/seqid=0 2024-11-17T07:13:48,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741889_1065 (size=12301) 2024-11-17T07:13:48,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T07:13:48,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:48,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:48,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:48,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:48,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:49,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:49,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827689047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:49,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,059 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:49,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:49,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827689058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:49,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:49,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827689058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:49,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827689067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:49,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827689077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:49,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T07:13:49,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:49,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:49,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:49,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:49,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:49,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:49,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T07:13:49,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T07:13:49,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:49,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:49,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:49,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:49,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:49,258 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/88a065ba61404bc18250c3f939fe4bf3 2024-11-17T07:13:49,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/9235a4bfd3c045d7a028ff0cde01a5f8 is 50, key is test_row_0/C:col10/1731827627908/Put/seqid=0 2024-11-17T07:13:49,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741890_1066 (size=12301) 2024-11-17T07:13:49,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/9235a4bfd3c045d7a028ff0cde01a5f8 2024-11-17T07:13:49,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/baaf67a34abf4558b4001fb4ccfa29f3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/baaf67a34abf4558b4001fb4ccfa29f3 2024-11-17T07:13:49,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/baaf67a34abf4558b4001fb4ccfa29f3, entries=150, sequenceid=377, filesize=12.0 K 2024-11-17T07:13:49,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/88a065ba61404bc18250c3f939fe4bf3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/88a065ba61404bc18250c3f939fe4bf3 2024-11-17T07:13:49,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/88a065ba61404bc18250c3f939fe4bf3, entries=150, sequenceid=377, filesize=12.0 K 2024-11-17T07:13:49,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/9235a4bfd3c045d7a028ff0cde01a5f8 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9235a4bfd3c045d7a028ff0cde01a5f8 2024-11-17T07:13:49,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9235a4bfd3c045d7a028ff0cde01a5f8, entries=150, sequenceid=377, filesize=12.0 K 2024-11-17T07:13:49,336 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for cccbc0d9c2f0bc3781663c8c9bb7223f in 916ms, sequenceid=377, compaction requested=true 2024-11-17T07:13:49,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:49,336 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:13:49,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:13:49,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:49,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:13:49,337 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:13:49,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:49,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:13:49,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:49,340 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52122 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:13:49,340 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/A is initiating minor compaction (all files) 2024-11-17T07:13:49,340 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/A in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:49,340 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/b833a96f35aa452ca57d3fc76a292f04, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/c1bd9ff043e34fd09df31240e3c92b85, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a6e832f36f494a29bbc121901e3e8c12, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/baaf67a34abf4558b4001fb4ccfa29f3] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=50.9 K 2024-11-17T07:13:49,341 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:13:49,341 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/B is initiating minor compaction (all files) 2024-11-17T07:13:49,341 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/B in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:49,341 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/4103d81e897840dfa681ddaa5771bd47, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/f6f2771ced354b88bc050db42db67e52, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/49856b44e4fe4abab7e1b205f0604798, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/88a065ba61404bc18250c3f939fe4bf3] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=48.5 K 2024-11-17T07:13:49,341 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting b833a96f35aa452ca57d3fc76a292f04, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731827626524 2024-11-17T07:13:49,342 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 4103d81e897840dfa681ddaa5771bd47, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731827626524 2024-11-17T07:13:49,342 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1bd9ff043e34fd09df31240e3c92b85, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731827626624 2024-11-17T07:13:49,343 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6e832f36f494a29bbc121901e3e8c12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731827627240 2024-11-17T07:13:49,343 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting f6f2771ced354b88bc050db42db67e52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731827626624 2024-11-17T07:13:49,343 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting baaf67a34abf4558b4001fb4ccfa29f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731827627908 2024-11-17T07:13:49,344 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 49856b44e4fe4abab7e1b205f0604798, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731827627240 2024-11-17T07:13:49,346 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 88a065ba61404bc18250c3f939fe4bf3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731827627908 2024-11-17T07:13:49,368 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#B#compaction#54 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:49,368 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/d543037de57f4261b35895d3e9247bd1 is 50, key is test_row_0/B:col10/1731827627908/Put/seqid=0 2024-11-17T07:13:49,375 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#A#compaction#55 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:49,376 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/a32f055771c4406c83de9c8c3e60b71b is 50, key is test_row_0/A:col10/1731827627908/Put/seqid=0 2024-11-17T07:13:49,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741891_1067 (size=12915) 2024-11-17T07:13:49,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-17T07:13:49,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:49,400 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-17T07:13:49,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:49,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:49,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:49,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:49,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:49,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:49,404 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/d543037de57f4261b35895d3e9247bd1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d543037de57f4261b35895d3e9247bd1 2024-11-17T07:13:49,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741892_1068 (size=12915) 2024-11-17T07:13:49,415 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/B of cccbc0d9c2f0bc3781663c8c9bb7223f into d543037de57f4261b35895d3e9247bd1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:49,415 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:49,415 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/B, priority=12, startTime=1731827629337; duration=0sec 2024-11-17T07:13:49,415 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:49,415 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:B 2024-11-17T07:13:49,416 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:13:49,418 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/a32f055771c4406c83de9c8c3e60b71b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a32f055771c4406c83de9c8c3e60b71b 2024-11-17T07:13:49,419 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:13:49,419 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/C is initiating minor compaction (all files) 2024-11-17T07:13:49,419 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/C in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:49,419 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/14b39cb6dba6491a96ff132b7847af0d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/78ebaeddc73b4a03986ed067088a1561, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/0c39f90a90c2437cb71f56dfa587b671, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9235a4bfd3c045d7a028ff0cde01a5f8] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=48.5 K 2024-11-17T07:13:49,420 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 14b39cb6dba6491a96ff132b7847af0d, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731827626524 2024-11-17T07:13:49,420 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 78ebaeddc73b4a03986ed067088a1561, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731827626624 2024-11-17T07:13:49,421 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c39f90a90c2437cb71f56dfa587b671, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731827627240 2024-11-17T07:13:49,421 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 9235a4bfd3c045d7a028ff0cde01a5f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731827627908 2024-11-17T07:13:49,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/567cba1c294f4923ac4c30a94d05bcae is 50, key is test_row_0/A:col10/1731827628431/Put/seqid=0 2024-11-17T07:13:49,426 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/A of cccbc0d9c2f0bc3781663c8c9bb7223f into a32f055771c4406c83de9c8c3e60b71b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:49,426 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:49,426 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/A, priority=12, startTime=1731827629336; duration=0sec 2024-11-17T07:13:49,426 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:49,426 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:A 2024-11-17T07:13:49,458 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#C#compaction#57 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:49,459 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/3eb5014dd9634a319f79cafa921266d7 is 50, key is test_row_0/C:col10/1731827627908/Put/seqid=0 2024-11-17T07:13:49,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741893_1069 (size=12301) 2024-11-17T07:13:49,464 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/567cba1c294f4923ac4c30a94d05bcae 2024-11-17T07:13:49,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/d26f1f705cc747278f7a53206b50d539 is 50, key is test_row_0/B:col10/1731827628431/Put/seqid=0 2024-11-17T07:13:49,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741894_1070 (size=12915) 2024-11-17T07:13:49,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741895_1071 (size=12301) 2024-11-17T07:13:49,507 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/3eb5014dd9634a319f79cafa921266d7 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/3eb5014dd9634a319f79cafa921266d7 2024-11-17T07:13:49,518 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/C of cccbc0d9c2f0bc3781663c8c9bb7223f into 3eb5014dd9634a319f79cafa921266d7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:49,518 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:49,518 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/C, priority=12, startTime=1731827629339; duration=0sec 2024-11-17T07:13:49,519 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:49,519 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:C 2024-11-17T07:13:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:49,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T07:13:49,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827629634, max timestamp=1731827629676Num uniques 859; 2024-11-17T07:13:49,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:13:49,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827629634, max timestamp=1731827629676Num uniques 859; 2024-11-17T07:13:49,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:13:49,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827629634, max timestamp=1731827629676Num uniques 859; 2024-11-17T07:13:49,682 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:13:49,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827689768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:49,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:49,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827689877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:49,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:49,895 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/d26f1f705cc747278f7a53206b50d539 2024-11-17T07:13:49,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/afde37dfb9524673a1877ac1448dc3a3 is 50, key is test_row_0/C:col10/1731827628431/Put/seqid=0 2024-11-17T07:13:49,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741896_1072 (size=12301) 2024-11-17T07:13:50,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827690068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:50,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827690068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:50,074 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in cccbc0d9c2f0bc3781663c8c9bb7223f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:13:50,074 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in cccbc0d9c2f0bc3781663c8c9bb7223f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:13:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:50,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827690077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:50,078 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cccbc0d9c2f0bc3781663c8c9bb7223f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:13:50,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:50,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:50,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827690087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:50,090 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in cccbc0d9c2f0bc3781663c8c9bb7223f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:13:50,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:50,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:50,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827690088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,092 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:50,327 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=386 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/afde37dfb9524673a1877ac1448dc3a3 2024-11-17T07:13:50,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/567cba1c294f4923ac4c30a94d05bcae as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/567cba1c294f4923ac4c30a94d05bcae 2024-11-17T07:13:50,340 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/567cba1c294f4923ac4c30a94d05bcae, entries=150, sequenceid=386, filesize=12.0 K 2024-11-17T07:13:50,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/d26f1f705cc747278f7a53206b50d539 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d26f1f705cc747278f7a53206b50d539 2024-11-17T07:13:50,346 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d26f1f705cc747278f7a53206b50d539, entries=150, sequenceid=386, filesize=12.0 K 2024-11-17T07:13:50,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/afde37dfb9524673a1877ac1448dc3a3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/afde37dfb9524673a1877ac1448dc3a3 2024-11-17T07:13:50,353 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/afde37dfb9524673a1877ac1448dc3a3, entries=150, sequenceid=386, filesize=12.0 K 2024-11-17T07:13:50,354 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for cccbc0d9c2f0bc3781663c8c9bb7223f in 954ms, sequenceid=386, compaction requested=false 2024-11-17T07:13:50,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:50,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:50,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-17T07:13:50,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-17T07:13:50,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-17T07:13:50,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8840 sec 2024-11-17T07:13:50,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.8910 sec 2024-11-17T07:13:50,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:50,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-11-17T07:13:50,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:50,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:50,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:50,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:50,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:50,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:50,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/5657e2e8a8c94520872c2ada618e3ec9 is 50, key is test_row_0/A:col10/1731827629757/Put/seqid=0 2024-11-17T07:13:50,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741897_1073 (size=14741) 2024-11-17T07:13:50,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:50,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827690426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:50,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:50,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827690537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-17T07:13:50,618 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:13:50,618 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:13:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-17T07:13:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T07:13:50,622 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:13:50,623 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:13:50,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:13:50,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T07:13:50,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:50,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827690747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:50,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:50,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T07:13:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:50,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:50,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:50,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/5657e2e8a8c94520872c2ada618e3ec9 2024-11-17T07:13:50,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/e01c41b5b8954c82a58e4e134e9f7f54 is 50, key is test_row_0/B:col10/1731827629757/Put/seqid=0 2024-11-17T07:13:50,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741898_1074 (size=12301) 2024-11-17T07:13:50,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T07:13:50,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T07:13:50,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:50,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:50,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:50,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:50,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827691057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:51,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T07:13:51,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:51,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/e01c41b5b8954c82a58e4e134e9f7f54 2024-11-17T07:13:51,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T07:13:51,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T07:13:51,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/afddaf9aa48140aeb851621566fdfbcf is 50, key is test_row_0/C:col10/1731827629757/Put/seqid=0 2024-11-17T07:13:51,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741899_1075 (size=12301) 2024-11-17T07:13:51,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T07:13:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:51,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T07:13:51,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:51,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827691567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:51,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/afddaf9aa48140aeb851621566fdfbcf 2024-11-17T07:13:51,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/5657e2e8a8c94520872c2ada618e3ec9 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5657e2e8a8c94520872c2ada618e3ec9 2024-11-17T07:13:51,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5657e2e8a8c94520872c2ada618e3ec9, entries=200, sequenceid=426, filesize=14.4 K 2024-11-17T07:13:51,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/e01c41b5b8954c82a58e4e134e9f7f54 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/e01c41b5b8954c82a58e4e134e9f7f54 2024-11-17T07:13:51,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/e01c41b5b8954c82a58e4e134e9f7f54, entries=150, sequenceid=426, filesize=12.0 K 2024-11-17T07:13:51,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/afddaf9aa48140aeb851621566fdfbcf as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/afddaf9aa48140aeb851621566fdfbcf 2024-11-17T07:13:51,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/afddaf9aa48140aeb851621566fdfbcf, entries=150, sequenceid=426, filesize=12.0 K 2024-11-17T07:13:51,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T07:13:51,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:51,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:51,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for cccbc0d9c2f0bc3781663c8c9bb7223f in 1312ms, sequenceid=426, compaction requested=true 2024-11-17T07:13:51,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:51,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:13:51,711 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:51,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:51,711 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:51,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:13:51,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:51,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:13:51,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:51,714 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:51,714 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/A is initiating minor compaction (all files) 2024-11-17T07:13:51,714 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/A in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,714 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a32f055771c4406c83de9c8c3e60b71b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/567cba1c294f4923ac4c30a94d05bcae, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5657e2e8a8c94520872c2ada618e3ec9] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=39.0 K 2024-11-17T07:13:51,714 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:51,714 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/B is initiating minor compaction (all files) 2024-11-17T07:13:51,714 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/B in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,715 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d543037de57f4261b35895d3e9247bd1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d26f1f705cc747278f7a53206b50d539, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/e01c41b5b8954c82a58e4e134e9f7f54] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=36.6 K 2024-11-17T07:13:51,715 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting d543037de57f4261b35895d3e9247bd1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731827627908 2024-11-17T07:13:51,715 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting a32f055771c4406c83de9c8c3e60b71b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731827627908 2024-11-17T07:13:51,715 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting d26f1f705cc747278f7a53206b50d539, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731827628431 2024-11-17T07:13:51,716 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 567cba1c294f4923ac4c30a94d05bcae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731827628431 2024-11-17T07:13:51,716 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting e01c41b5b8954c82a58e4e134e9f7f54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731827629757 2024-11-17T07:13:51,717 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5657e2e8a8c94520872c2ada618e3ec9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731827629745 2024-11-17T07:13:51,730 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:51,731 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/7a5a187f502e47a191a2be95d25acf84 is 50, key is test_row_0/B:col10/1731827629757/Put/seqid=0 2024-11-17T07:13:51,739 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#A#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:51,740 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/cbbb4db557f54cc4ad533d7c01c96f09 is 50, key is test_row_0/A:col10/1731827629757/Put/seqid=0 2024-11-17T07:13:51,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741900_1076 (size=13017) 2024-11-17T07:13:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T07:13:51,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741901_1077 (size=13017) 2024-11-17T07:13:51,779 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/cbbb4db557f54cc4ad533d7c01c96f09 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cbbb4db557f54cc4ad533d7c01c96f09 2024-11-17T07:13:51,794 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/A of cccbc0d9c2f0bc3781663c8c9bb7223f into cbbb4db557f54cc4ad533d7c01c96f09(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:51,794 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:51,794 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/A, priority=13, startTime=1731827631711; duration=0sec 2024-11-17T07:13:51,794 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:51,794 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:A 2024-11-17T07:13:51,794 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:51,796 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:51,796 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/C is initiating minor compaction (all files) 2024-11-17T07:13:51,796 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/C in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,796 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/3eb5014dd9634a319f79cafa921266d7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/afde37dfb9524673a1877ac1448dc3a3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/afddaf9aa48140aeb851621566fdfbcf] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=36.6 K 2024-11-17T07:13:51,797 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eb5014dd9634a319f79cafa921266d7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1731827627908 2024-11-17T07:13:51,798 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting afde37dfb9524673a1877ac1448dc3a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=386, earliestPutTs=1731827628431 2024-11-17T07:13:51,798 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting afddaf9aa48140aeb851621566fdfbcf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731827629757 2024-11-17T07:13:51,817 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#C#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:51,818 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/9fbd4084f71b43dcbc9d40f7498811a6 is 50, key is test_row_0/C:col10/1731827629757/Put/seqid=0 2024-11-17T07:13:51,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741902_1078 (size=13017) 2024-11-17T07:13:51,835 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/9fbd4084f71b43dcbc9d40f7498811a6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9fbd4084f71b43dcbc9d40f7498811a6 2024-11-17T07:13:51,853 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/C of cccbc0d9c2f0bc3781663c8c9bb7223f into 9fbd4084f71b43dcbc9d40f7498811a6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:51,853 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:51,853 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/C, priority=13, startTime=1731827631712; duration=0sec 2024-11-17T07:13:51,853 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:51,853 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:C 2024-11-17T07:13:51,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-17T07:13:51,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:51,864 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T07:13:51,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:51,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:51,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:51,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:51,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:51,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:51,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/ab31d3c3042040dfb80c60f990c46623 is 50, key is test_row_0/A:col10/1731827630415/Put/seqid=0 2024-11-17T07:13:51,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741903_1079 (size=12301) 2024-11-17T07:13:51,895 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/ab31d3c3042040dfb80c60f990c46623 2024-11-17T07:13:51,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/52cd311ae2324ff6ab791dc4b0783f6a is 50, key is test_row_0/B:col10/1731827630415/Put/seqid=0 2024-11-17T07:13:51,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741904_1080 (size=12301) 2024-11-17T07:13:52,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:52,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:52,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731827632112, max timestamp=1731827632124Num uniques 859; 2024-11-17T07:13:52,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T07:13:52,130 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731827632112, max timestamp=1731827632124Num uniques 859; 2024-11-17T07:13:52,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731827632112, max timestamp=1731827632124Num uniques 859; 2024-11-17T07:13:52,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T07:13:52,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T07:13:52,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827692142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827692143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,145 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827692144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827692144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,160 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/7a5a187f502e47a191a2be95d25acf84 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7a5a187f502e47a191a2be95d25acf84 2024-11-17T07:13:52,168 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/B of cccbc0d9c2f0bc3781663c8c9bb7223f into 7a5a187f502e47a191a2be95d25acf84(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:52,168 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:52,169 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/B, priority=13, startTime=1731827631711; duration=0sec 2024-11-17T07:13:52,169 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:52,169 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:B 2024-11-17T07:13:52,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827692247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827692247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827692258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827692258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,323 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/52cd311ae2324ff6ab791dc4b0783f6a 2024-11-17T07:13:52,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/782f87993fb146c99d32a04c16624511 is 50, key is test_row_0/C:col10/1731827630415/Put/seqid=0 2024-11-17T07:13:52,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741905_1081 (size=12301) 2024-11-17T07:13:52,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827692457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827692458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,465 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827692467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827692467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827692577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,579 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in cccbc0d9c2f0bc3781663c8c9bb7223f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:13:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,737 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=436 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/782f87993fb146c99d32a04c16624511 2024-11-17T07:13:52,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/ab31d3c3042040dfb80c60f990c46623 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ab31d3c3042040dfb80c60f990c46623 2024-11-17T07:13:52,754 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ab31d3c3042040dfb80c60f990c46623, entries=150, sequenceid=436, filesize=12.0 K 2024-11-17T07:13:52,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/52cd311ae2324ff6ab791dc4b0783f6a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/52cd311ae2324ff6ab791dc4b0783f6a 2024-11-17T07:13:52,765 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/52cd311ae2324ff6ab791dc4b0783f6a, entries=150, sequenceid=436, filesize=12.0 K 2024-11-17T07:13:52,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/782f87993fb146c99d32a04c16624511 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/782f87993fb146c99d32a04c16624511 2024-11-17T07:13:52,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T07:13:52,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827692767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827692777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827692778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,781 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/782f87993fb146c99d32a04c16624511, entries=150, sequenceid=436, filesize=12.0 K 2024-11-17T07:13:52,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:52,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827692778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,782 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=161.02 KB/164880 for cccbc0d9c2f0bc3781663c8c9bb7223f in 918ms, sequenceid=436, compaction requested=false 2024-11-17T07:13:52,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:52,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:52,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-17T07:13:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-17T07:13:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:52,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-17T07:13:52,786 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1610 sec 2024-11-17T07:13:52,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.1690 sec 2024-11-17T07:13:53,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:53,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=167.72 KB heapSize=466.55 KB 2024-11-17T07:13:53,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:53,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:13:53,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:53,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:13:53,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:53,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:13:53,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/f66384ea5fda4e1f9625a44cd53d518b is 50, key is test_row_0/A:col10/1731827632140/Put/seqid=0 2024-11-17T07:13:53,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:53,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741906_1082 (size=14741) 2024-11-17T07:13:53,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827693291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:53,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:53,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/f66384ea5fda4e1f9625a44cd53d518b 2024-11-17T07:13:53,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:53,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827693296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827693297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:53,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:53,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827693299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:53,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/1375cd3adb034d8c9a4e26df6ef79b53 is 50, key is test_row_0/B:col10/1731827632140/Put/seqid=0 2024-11-17T07:13:53,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741907_1083 (size=12301) 2024-11-17T07:13:53,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/1375cd3adb034d8c9a4e26df6ef79b53 2024-11-17T07:13:53,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/6d6bbc7731c245f0b3e18ffc9f4823a1 is 50, key is test_row_0/C:col10/1731827632140/Put/seqid=0 2024-11-17T07:13:53,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741908_1084 (size=12301) 2024-11-17T07:13:53,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:53,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827693408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:53,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827693408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:53,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:53,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827693417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:53,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827693417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:53,421 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:53,623 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:53,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827693617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:53,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:53,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:53,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827693617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:53,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:53,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:53,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827693628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:53,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:53,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827693628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:53,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:53,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=475 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/6d6bbc7731c245f0b3e18ffc9f4823a1 2024-11-17T07:13:53,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/f66384ea5fda4e1f9625a44cd53d518b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f66384ea5fda4e1f9625a44cd53d518b 2024-11-17T07:13:53,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f66384ea5fda4e1f9625a44cd53d518b, entries=200, sequenceid=475, filesize=14.4 K 2024-11-17T07:13:53,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/1375cd3adb034d8c9a4e26df6ef79b53 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/1375cd3adb034d8c9a4e26df6ef79b53 2024-11-17T07:13:53,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/1375cd3adb034d8c9a4e26df6ef79b53, entries=150, sequenceid=475, filesize=12.0 K 2024-11-17T07:13:53,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/6d6bbc7731c245f0b3e18ffc9f4823a1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/6d6bbc7731c245f0b3e18ffc9f4823a1 2024-11-17T07:13:53,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/6d6bbc7731c245f0b3e18ffc9f4823a1, entries=150, sequenceid=475, filesize=12.0 K 2024-11-17T07:13:53,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~466.50 KB/477696, currentSize=26.84 KB/27480 for cccbc0d9c2f0bc3781663c8c9bb7223f in 579ms, sequenceid=475, compaction requested=true 2024-11-17T07:13:53,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:53,858 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:53,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:13:53,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:53,859 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:53,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:13:53,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:53,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:13:53,859 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:53,860 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:53,860 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/A is initiating minor compaction (all files) 2024-11-17T07:13:53,860 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/A in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:53,860 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cbbb4db557f54cc4ad533d7c01c96f09, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ab31d3c3042040dfb80c60f990c46623, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f66384ea5fda4e1f9625a44cd53d518b] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=39.1 K 2024-11-17T07:13:53,861 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:53,861 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/B is initiating minor compaction (all files) 2024-11-17T07:13:53,861 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbbb4db557f54cc4ad533d7c01c96f09, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731827629757 2024-11-17T07:13:53,861 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/B in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:53,861 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7a5a187f502e47a191a2be95d25acf84, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/52cd311ae2324ff6ab791dc4b0783f6a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/1375cd3adb034d8c9a4e26df6ef79b53] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=36.7 K 2024-11-17T07:13:53,862 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab31d3c3042040dfb80c60f990c46623, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731827630404 2024-11-17T07:13:53,862 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a5a187f502e47a191a2be95d25acf84, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731827629757 2024-11-17T07:13:53,863 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting f66384ea5fda4e1f9625a44cd53d518b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731827632140 2024-11-17T07:13:53,863 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 52cd311ae2324ff6ab791dc4b0783f6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731827630404 2024-11-17T07:13:53,863 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 1375cd3adb034d8c9a4e26df6ef79b53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731827632140 2024-11-17T07:13:53,882 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:53,883 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/9d627176b551498eb3ffbff8b50388ee is 50, key is test_row_0/B:col10/1731827632140/Put/seqid=0 2024-11-17T07:13:53,886 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:53,887 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/aaa31809583a449981cf390cb0cd9f77 is 50, key is test_row_0/A:col10/1731827632140/Put/seqid=0 2024-11-17T07:13:53,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741909_1085 (size=13119) 2024-11-17T07:13:53,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741910_1086 (size=13119) 2024-11-17T07:13:53,931 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/aaa31809583a449981cf390cb0cd9f77 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/aaa31809583a449981cf390cb0cd9f77 2024-11-17T07:13:53,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:53,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:13:53,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:53,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:53,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:53,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:53,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:53,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:53,945 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/A of cccbc0d9c2f0bc3781663c8c9bb7223f into aaa31809583a449981cf390cb0cd9f77(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:53,945 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:53,945 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/A, priority=13, startTime=1731827633858; duration=0sec 2024-11-17T07:13:53,945 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:53,945 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:A 2024-11-17T07:13:53,945 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:53,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/cf48b7519fd548beabf7885bd64a559a is 50, key is test_row_0/A:col10/1731827633938/Put/seqid=0 2024-11-17T07:13:53,948 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:53,948 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/C is initiating minor compaction (all files) 2024-11-17T07:13:53,948 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/C in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:53,948 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9fbd4084f71b43dcbc9d40f7498811a6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/782f87993fb146c99d32a04c16624511, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/6d6bbc7731c245f0b3e18ffc9f4823a1] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=36.7 K 2024-11-17T07:13:53,950 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fbd4084f71b43dcbc9d40f7498811a6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731827629757 2024-11-17T07:13:53,952 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 782f87993fb146c99d32a04c16624511, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=436, earliestPutTs=1731827630404 2024-11-17T07:13:53,953 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d6bbc7731c245f0b3e18ffc9f4823a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731827632140 2024-11-17T07:13:53,972 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#C#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:53,973 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/87590ac893f94b30a83b4aa8855c3109 is 50, key is test_row_0/C:col10/1731827632140/Put/seqid=0 2024-11-17T07:13:53,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741911_1087 (size=17181) 2024-11-17T07:13:53,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731827633959, max timestamp=1731827633980Num uniques 859; 2024-11-17T07:13:53,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731827633959, max timestamp=1731827633980Num uniques 859; 2024-11-17T07:13:53,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-17T07:13:53,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-17T07:13:53,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827633967, max timestamp=1731827633980Num uniques 859; 2024-11-17T07:13:53,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:53,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/cf48b7519fd548beabf7885bd64a559a 2024-11-17T07:13:53,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741912_1088 (size=13119) 2024-11-17T07:13:53,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/268e9ea3ee96402e81b1e8d306c2a730 is 50, key is test_row_0/B:col10/1731827633938/Put/seqid=0 2024-11-17T07:13:54,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741913_1089 (size=12301) 2024-11-17T07:13:54,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827694006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827694009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827694010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827694011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827694117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,123 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827694118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827694127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827694127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,131 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,319 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/9d627176b551498eb3ffbff8b50388ee as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/9d627176b551498eb3ffbff8b50388ee 2024-11-17T07:13:54,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827694328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827694328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,334 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/B of cccbc0d9c2f0bc3781663c8c9bb7223f into 9d627176b551498eb3ffbff8b50388ee(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:54,334 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:54,334 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/B, priority=13, startTime=1731827633858; duration=0sec 2024-11-17T07:13:54,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827694337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,340 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:54,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827694337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,340 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:B 2024-11-17T07:13:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,396 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/87590ac893f94b30a83b4aa8855c3109 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/87590ac893f94b30a83b4aa8855c3109 2024-11-17T07:13:54,402 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/C of cccbc0d9c2f0bc3781663c8c9bb7223f into 87590ac893f94b30a83b4aa8855c3109(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:54,402 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:54,402 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/C, priority=13, startTime=1731827633859; duration=0sec 2024-11-17T07:13:54,402 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:54,403 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:C 2024-11-17T07:13:54,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/268e9ea3ee96402e81b1e8d306c2a730 2024-11-17T07:13:54,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/eaee706b17d540f88f31eeebaa4928e5 is 50, key is test_row_0/C:col10/1731827633938/Put/seqid=0 2024-11-17T07:13:54,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741914_1090 (size=12301) 2024-11-17T07:13:54,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/eaee706b17d540f88f31eeebaa4928e5 2024-11-17T07:13:54,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/cf48b7519fd548beabf7885bd64a559a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cf48b7519fd548beabf7885bd64a559a 2024-11-17T07:13:54,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cf48b7519fd548beabf7885bd64a559a, entries=250, sequenceid=485, filesize=16.8 K 2024-11-17T07:13:54,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/268e9ea3ee96402e81b1e8d306c2a730 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/268e9ea3ee96402e81b1e8d306c2a730 2024-11-17T07:13:54,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/268e9ea3ee96402e81b1e8d306c2a730, entries=150, sequenceid=485, filesize=12.0 K 2024-11-17T07:13:54,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/eaee706b17d540f88f31eeebaa4928e5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/eaee706b17d540f88f31eeebaa4928e5 2024-11-17T07:13:54,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/eaee706b17d540f88f31eeebaa4928e5, entries=150, sequenceid=485, filesize=12.0 K 2024-11-17T07:13:54,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=136.42 KB/139690 for cccbc0d9c2f0bc3781663c8c9bb7223f in 517ms, sequenceid=485, compaction requested=false 2024-11-17T07:13:54,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:54,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=143.13 KB heapSize=395.92 KB 2024-11-17T07:13:54,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:54,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:54,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:54,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:54,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:54,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:54,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/23ab2450ccd5402f806e55a7b07a91ee is 50, key is test_row_0/A:col10/1731827634589/Put/seqid=0 2024-11-17T07:13:54,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741915_1091 (size=14741) 2024-11-17T07:13:54,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827694620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827694637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827694637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827694647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827694647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827694727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-17T07:13:54,792 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:13:54,792 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:13:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-17T07:13:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T07:13:54,794 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:13:54,795 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:13:54,795 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:13:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T07:13:54,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827694937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:54,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:54,940 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:54,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T07:13:54,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:54,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:54,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:54,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:54,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:55,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/23ab2450ccd5402f806e55a7b07a91ee 2024-11-17T07:13:55,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/839842ba29754f7691637ba1de8cc902 is 50, key is test_row_0/B:col10/1731827634589/Put/seqid=0 2024-11-17T07:13:55,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741916_1092 (size=12301) 2024-11-17T07:13:55,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/839842ba29754f7691637ba1de8cc902 2024-11-17T07:13:55,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/30797e3f6415468fabf4122996c3b84c is 50, key is test_row_0/C:col10/1731827634589/Put/seqid=0 2024-11-17T07:13:55,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741917_1093 (size=12301) 2024-11-17T07:13:55,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T07:13:55,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:55,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:55,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:55,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:55,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:55,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T07:13:55,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827695147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:55,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:55,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:55,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:55,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827695157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:55,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827695157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:55,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:55,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827695157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:55,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:55,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:55,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827695247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:55,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:55,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T07:13:55,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:55,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:55,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:55,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:55,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:55,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T07:13:55,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:55,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:55,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:55,409 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:55,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T07:13:55,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/30797e3f6415468fabf4122996c3b84c 2024-11-17T07:13:55,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/23ab2450ccd5402f806e55a7b07a91ee as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/23ab2450ccd5402f806e55a7b07a91ee 2024-11-17T07:13:55,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/23ab2450ccd5402f806e55a7b07a91ee, entries=200, sequenceid=523, filesize=14.4 K 2024-11-17T07:13:55,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/839842ba29754f7691637ba1de8cc902 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/839842ba29754f7691637ba1de8cc902 2024-11-17T07:13:55,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/839842ba29754f7691637ba1de8cc902, entries=150, sequenceid=523, filesize=12.0 K 2024-11-17T07:13:55,469 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/30797e3f6415468fabf4122996c3b84c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/30797e3f6415468fabf4122996c3b84c 2024-11-17T07:13:55,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/30797e3f6415468fabf4122996c3b84c, entries=150, sequenceid=523, filesize=12.0 K 2024-11-17T07:13:55,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~395.88 KB/405376, currentSize=40.25 KB/41220 for cccbc0d9c2f0bc3781663c8c9bb7223f in 891ms, sequenceid=523, compaction requested=true 2024-11-17T07:13:55,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:55,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:13:55,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:55,483 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:55,483 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:55,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:13:55,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:55,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:13:55,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:55,484 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:55,484 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/A is initiating minor compaction (all files) 2024-11-17T07:13:55,484 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/A in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:55,484 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/aaa31809583a449981cf390cb0cd9f77, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cf48b7519fd548beabf7885bd64a559a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/23ab2450ccd5402f806e55a7b07a91ee] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=44.0 K 2024-11-17T07:13:55,485 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:55,485 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/B is initiating minor compaction (all files) 2024-11-17T07:13:55,485 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/B in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:55,485 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaa31809583a449981cf390cb0cd9f77, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731827632140 2024-11-17T07:13:55,485 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/9d627176b551498eb3ffbff8b50388ee, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/268e9ea3ee96402e81b1e8d306c2a730, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/839842ba29754f7691637ba1de8cc902] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=36.8 K 2024-11-17T07:13:55,485 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf48b7519fd548beabf7885bd64a559a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731827633292 2024-11-17T07:13:55,486 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d627176b551498eb3ffbff8b50388ee, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731827632140 2024-11-17T07:13:55,486 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23ab2450ccd5402f806e55a7b07a91ee, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731827634006 2024-11-17T07:13:55,486 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 268e9ea3ee96402e81b1e8d306c2a730, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731827633294 2024-11-17T07:13:55,486 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 839842ba29754f7691637ba1de8cc902, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731827634006 2024-11-17T07:13:55,495 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#A#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:55,496 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/f2c7a4814b634775bddf0a3697a47f4f is 50, key is test_row_0/A:col10/1731827634589/Put/seqid=0 2024-11-17T07:13:55,502 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#B#compaction#82 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:55,502 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/fe0998c5ed04454da2c8f327f3ee97b3 is 50, key is test_row_0/B:col10/1731827634589/Put/seqid=0 2024-11-17T07:13:55,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741918_1094 (size=13221) 2024-11-17T07:13:55,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741919_1095 (size=13221) 2024-11-17T07:13:55,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-17T07:13:55,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:55,564 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-17T07:13:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:55,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:55,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/8005d19361fb4ecf8127b161006b1e3d is 50, key is test_row_0/A:col10/1731827634613/Put/seqid=0 2024-11-17T07:13:55,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741920_1096 (size=12301) 2024-11-17T07:13:55,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:55,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:55,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827635816, max timestamp=1731827635828Num uniques 859; 2024-11-17T07:13:55,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:55,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827635816, max timestamp=1731827635828Num uniques 859; 2024-11-17T07:13:55,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827635816, max timestamp=1731827635828Num uniques 859; 2024-11-17T07:13:55,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:55,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:55,918 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/f2c7a4814b634775bddf0a3697a47f4f as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f2c7a4814b634775bddf0a3697a47f4f 2024-11-17T07:13:55,920 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/fe0998c5ed04454da2c8f327f3ee97b3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/fe0998c5ed04454da2c8f327f3ee97b3 2024-11-17T07:13:55,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T07:13:55,927 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/A of cccbc0d9c2f0bc3781663c8c9bb7223f into f2c7a4814b634775bddf0a3697a47f4f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:55,927 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:55,927 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/A, priority=13, startTime=1731827635482; duration=0sec 2024-11-17T07:13:55,928 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:55,928 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:A 2024-11-17T07:13:55,928 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:55,930 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:55,930 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/C is initiating minor compaction (all files) 2024-11-17T07:13:55,931 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/C in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:55,931 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/87590ac893f94b30a83b4aa8855c3109, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/eaee706b17d540f88f31eeebaa4928e5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/30797e3f6415468fabf4122996c3b84c] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=36.8 K 2024-11-17T07:13:55,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731827635859, max timestamp=1731827635929Num uniques 1018; 2024-11-17T07:13:55,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-17T07:13:55,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731827635859, max timestamp=1731827635929Num uniques 1018; 2024-11-17T07:13:55,934 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-17T07:13:55,934 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87590ac893f94b30a83b4aa8855c3109, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=475, earliestPutTs=1731827632140 2024-11-17T07:13:55,935 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaee706b17d540f88f31eeebaa4928e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731827633294 2024-11-17T07:13:55,935 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/B of cccbc0d9c2f0bc3781663c8c9bb7223f into fe0998c5ed04454da2c8f327f3ee97b3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:55,935 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:55,935 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/B, priority=13, startTime=1731827635483; duration=0sec 2024-11-17T07:13:55,935 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:55,935 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:B 2024-11-17T07:13:55,935 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30797e3f6415468fabf4122996c3b84c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731827634006 2024-11-17T07:13:55,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731827635859, max timestamp=1731827635929Num uniques 1018; 2024-11-17T07:13:55,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T07:13:55,946 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#C#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:55,948 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/04dacb77c83847f48e3b6d7e5127bfc1 is 50, key is test_row_0/C:col10/1731827634589/Put/seqid=0 2024-11-17T07:13:55,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741921_1097 (size=13221) 2024-11-17T07:13:55,976 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/04dacb77c83847f48e3b6d7e5127bfc1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/04dacb77c83847f48e3b6d7e5127bfc1 2024-11-17T07:13:55,977 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/8005d19361fb4ecf8127b161006b1e3d 2024-11-17T07:13:55,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/294edc772665413fa2578676b0332a54 is 50, key is test_row_0/B:col10/1731827634613/Put/seqid=0 2024-11-17T07:13:55,989 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/C of cccbc0d9c2f0bc3781663c8c9bb7223f into 04dacb77c83847f48e3b6d7e5127bfc1(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:55,989 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:55,989 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/C, priority=13, startTime=1731827635483; duration=0sec 2024-11-17T07:13:55,989 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:55,989 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:C 2024-11-17T07:13:55,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741922_1098 (size=12301) 2024-11-17T07:13:56,001 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/294edc772665413fa2578676b0332a54 2024-11-17T07:13:56,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/465853729bdf46b88f4b3423d0ecf346 is 50, key is test_row_0/C:col10/1731827634613/Put/seqid=0 2024-11-17T07:13:56,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741923_1099 (size=12301) 2024-11-17T07:13:56,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:56,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827696041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:56,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827696157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:56,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:56,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827696157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:56,164 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cccbc0d9c2f0bc3781663c8c9bb7223f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:13:56,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:56,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:56,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827696168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:56,170 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cccbc0d9c2f0bc3781663c8c9bb7223f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:13:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:56,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827696178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:56,179 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cccbc0d9c2f0bc3781663c8c9bb7223f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:13:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:56,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:56,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827696178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:56,183 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cccbc0d9c2f0bc3781663c8c9bb7223f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:13:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:56,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827696367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:56,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:56,439 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=532 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/465853729bdf46b88f4b3423d0ecf346 2024-11-17T07:13:56,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/8005d19361fb4ecf8127b161006b1e3d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/8005d19361fb4ecf8127b161006b1e3d 2024-11-17T07:13:56,457 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/8005d19361fb4ecf8127b161006b1e3d, entries=150, sequenceid=532, filesize=12.0 K 2024-11-17T07:13:56,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/294edc772665413fa2578676b0332a54 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/294edc772665413fa2578676b0332a54 2024-11-17T07:13:56,467 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/294edc772665413fa2578676b0332a54, entries=150, sequenceid=532, filesize=12.0 K 2024-11-17T07:13:56,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/465853729bdf46b88f4b3423d0ecf346 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/465853729bdf46b88f4b3423d0ecf346 2024-11-17T07:13:56,476 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/465853729bdf46b88f4b3423d0ecf346, entries=150, sequenceid=532, filesize=12.0 K 2024-11-17T07:13:56,477 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=143.13 KB/146560 for cccbc0d9c2f0bc3781663c8c9bb7223f in 913ms, sequenceid=532, compaction requested=false 2024-11-17T07:13:56,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:56,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:56,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-17T07:13:56,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-17T07:13:56,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-17T07:13:56,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6830 sec 2024-11-17T07:13:56,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.6890 sec 2024-11-17T07:13:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:56,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=149.83 KB heapSize=425.08 KB 2024-11-17T07:13:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:13:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:13:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:56,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:13:56,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/80ce2c406c38409c8ad517c1e6a2c1a6 is 50, key is test_row_0/A:col10/1731827636678/Put/seqid=0 2024-11-17T07:13:56,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741924_1100 (size=14741) 2024-11-17T07:13:56,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827696706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:56,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827696817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:56,818 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:56,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-17T07:13:56,937 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:13:56,938 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:13:56,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-17T07:13:56,941 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:13:56,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T07:13:56,944 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:13:56,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:13:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:57,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827697027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:57,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T07:13:57,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/80ce2c406c38409c8ad517c1e6a2c1a6 2024-11-17T07:13:57,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T07:13:57,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:57,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:57,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:57,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:57,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:57,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/7f7cf1bdb66845ed99c0070ba01ac7de is 50, key is test_row_0/B:col10/1731827636678/Put/seqid=0 2024-11-17T07:13:57,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741925_1101 (size=12301) 2024-11-17T07:13:57,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/7f7cf1bdb66845ed99c0070ba01ac7de 2024-11-17T07:13:57,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/03cd2c51e9ab41adbdf0c5daaea3c9a6 is 50, key is test_row_0/C:col10/1731827636678/Put/seqid=0 2024-11-17T07:13:57,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741926_1102 (size=12301) 2024-11-17T07:13:57,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=590 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/03cd2c51e9ab41adbdf0c5daaea3c9a6 2024-11-17T07:13:57,180 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/80ce2c406c38409c8ad517c1e6a2c1a6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/80ce2c406c38409c8ad517c1e6a2c1a6 2024-11-17T07:13:57,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/80ce2c406c38409c8ad517c1e6a2c1a6, entries=200, sequenceid=590, filesize=14.4 K 2024-11-17T07:13:57,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/7f7cf1bdb66845ed99c0070ba01ac7de as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7f7cf1bdb66845ed99c0070ba01ac7de 2024-11-17T07:13:57,192 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7f7cf1bdb66845ed99c0070ba01ac7de, entries=150, sequenceid=590, filesize=12.0 K 2024-11-17T07:13:57,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/03cd2c51e9ab41adbdf0c5daaea3c9a6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/03cd2c51e9ab41adbdf0c5daaea3c9a6 2024-11-17T07:13:57,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/03cd2c51e9ab41adbdf0c5daaea3c9a6, entries=150, sequenceid=590, filesize=12.0 K 2024-11-17T07:13:57,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~149.83 KB/153430, heapSize ~425.03 KB/435232, currentSize=33.54 KB/34350 for cccbc0d9c2f0bc3781663c8c9bb7223f in 519ms, sequenceid=590, compaction requested=true 2024-11-17T07:13:57,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:57,200 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:57,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:13:57,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:57,201 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:57,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:13:57,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:57,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:13:57,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:57,201 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:57,201 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/A is initiating minor compaction (all files) 2024-11-17T07:13:57,202 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/A in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:57,202 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f2c7a4814b634775bddf0a3697a47f4f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/8005d19361fb4ecf8127b161006b1e3d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/80ce2c406c38409c8ad517c1e6a2c1a6] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=39.3 K 2024-11-17T07:13:57,203 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2c7a4814b634775bddf0a3697a47f4f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731827634006 2024-11-17T07:13:57,203 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:57,203 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/B is initiating minor compaction (all files) 2024-11-17T07:13:57,203 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/B in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:57,203 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8005d19361fb4ecf8127b161006b1e3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731827634608 2024-11-17T07:13:57,203 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/fe0998c5ed04454da2c8f327f3ee97b3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/294edc772665413fa2578676b0332a54, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7f7cf1bdb66845ed99c0070ba01ac7de] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=36.9 K 2024-11-17T07:13:57,204 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80ce2c406c38409c8ad517c1e6a2c1a6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731827636015 2024-11-17T07:13:57,204 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting fe0998c5ed04454da2c8f327f3ee97b3, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731827634006 2024-11-17T07:13:57,206 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 294edc772665413fa2578676b0332a54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731827634608 2024-11-17T07:13:57,206 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f7cf1bdb66845ed99c0070ba01ac7de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731827636015 2024-11-17T07:13:57,230 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:57,231 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#B#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:57,231 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/7727fdb9c3f147f6b7a07a50c2bab19d is 50, key is test_row_0/A:col10/1731827636678/Put/seqid=0 2024-11-17T07:13:57,231 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/5fb20b956cf1412dbd53a25ca908db84 is 50, key is test_row_0/B:col10/1731827636678/Put/seqid=0 2024-11-17T07:13:57,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-17T07:13:57,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:57,254 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T07:13:57,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:57,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:57,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:57,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:57,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:57,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:13:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T07:13:57,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741928_1104 (size=13323) 2024-11-17T07:13:57,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/ad7eb9502a8d48e2964d5735f85b4907 is 50, key is test_row_1/A:col10/1731827636704/Put/seqid=0 2024-11-17T07:13:57,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741927_1103 (size=13323) 2024-11-17T07:13:57,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741929_1105 (size=9857) 2024-11-17T07:13:57,278 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/5fb20b956cf1412dbd53a25ca908db84 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/5fb20b956cf1412dbd53a25ca908db84 2024-11-17T07:13:57,289 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/B of cccbc0d9c2f0bc3781663c8c9bb7223f into 5fb20b956cf1412dbd53a25ca908db84(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:57,289 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:57,289 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/B, priority=13, startTime=1731827637201; duration=0sec 2024-11-17T07:13:57,289 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:57,289 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:B 2024-11-17T07:13:57,290 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:57,291 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:57,291 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/C is initiating minor compaction (all files) 2024-11-17T07:13:57,291 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/C in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:57,291 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/04dacb77c83847f48e3b6d7e5127bfc1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/465853729bdf46b88f4b3423d0ecf346, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/03cd2c51e9ab41adbdf0c5daaea3c9a6] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=36.9 K 2024-11-17T07:13:57,292 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 04dacb77c83847f48e3b6d7e5127bfc1, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731827634006 2024-11-17T07:13:57,292 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 465853729bdf46b88f4b3423d0ecf346, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=532, earliestPutTs=1731827634608 2024-11-17T07:13:57,293 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 03cd2c51e9ab41adbdf0c5daaea3c9a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731827636015 2024-11-17T07:13:57,307 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#C#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:57,308 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/13524d4862ce40389fc3996b288bf79e is 50, key is test_row_0/C:col10/1731827636678/Put/seqid=0 2024-11-17T07:13:57,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741930_1106 (size=13323) 2024-11-17T07:13:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:57,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:57,366 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/13524d4862ce40389fc3996b288bf79e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/13524d4862ce40389fc3996b288bf79e 2024-11-17T07:13:57,374 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/C of cccbc0d9c2f0bc3781663c8c9bb7223f into 13524d4862ce40389fc3996b288bf79e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:57,374 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:57,374 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/C, priority=13, startTime=1731827637201; duration=0sec 2024-11-17T07:13:57,375 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:57,375 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:C 2024-11-17T07:13:57,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827637413, max timestamp=1731827637430Num uniques 859; 2024-11-17T07:13:57,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:57,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827637413, max timestamp=1731827637430Num uniques 859; 2024-11-17T07:13:57,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:57,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827637413, max timestamp=1731827637430Num uniques 859; 2024-11-17T07:13:57,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:13:57,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731827637519, max timestamp=1731827637545Num uniques 1018; 2024-11-17T07:13:57,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-17T07:13:57,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731827637519, max timestamp=1731827637545Num uniques 1018; 2024-11-17T07:13:57,550 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-17T07:13:57,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731827637519, max timestamp=1731827637545Num uniques 1018; 2024-11-17T07:13:57,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T07:13:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T07:13:57,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731827637598, max timestamp=1731827637666Num uniques 1077; 2024-11-17T07:13:57,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-17T07:13:57,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731827637598, max timestamp=1731827637666Num uniques 1077; 2024-11-17T07:13:57,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-17T07:13:57,672 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/7727fdb9c3f147f6b7a07a50c2bab19d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7727fdb9c3f147f6b7a07a50c2bab19d 2024-11-17T07:13:57,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731827637598, max timestamp=1731827637666Num uniques 1027; 2024-11-17T07:13:57,675 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-17T07:13:57,678 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/ad7eb9502a8d48e2964d5735f85b4907 2024-11-17T07:13:57,683 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/A of cccbc0d9c2f0bc3781663c8c9bb7223f into 7727fdb9c3f147f6b7a07a50c2bab19d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:57,683 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:57,683 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/A, priority=13, startTime=1731827637200; duration=0sec 2024-11-17T07:13:57,683 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:57,683 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:A 2024-11-17T07:13:57,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/122ff82f2d5c4b5cb651bffa13378cd0 is 50, key is test_row_1/B:col10/1731827636704/Put/seqid=0 2024-11-17T07:13:57,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741931_1107 (size=9857) 2024-11-17T07:13:57,728 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/122ff82f2d5c4b5cb651bffa13378cd0 2024-11-17T07:13:57,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/78c0e65443214dfc9c15dc3d36ab7f22 is 50, key is test_row_1/C:col10/1731827636704/Put/seqid=0 2024-11-17T07:13:57,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741932_1108 (size=9857) 2024-11-17T07:13:57,769 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/78c0e65443214dfc9c15dc3d36ab7f22 2024-11-17T07:13:57,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/ad7eb9502a8d48e2964d5735f85b4907 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ad7eb9502a8d48e2964d5735f85b4907 2024-11-17T07:13:57,784 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ad7eb9502a8d48e2964d5735f85b4907, entries=100, sequenceid=598, filesize=9.6 K 2024-11-17T07:13:57,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/122ff82f2d5c4b5cb651bffa13378cd0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/122ff82f2d5c4b5cb651bffa13378cd0 2024-11-17T07:13:57,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731827637734, max timestamp=1731827637783Num uniques 1086; 2024-11-17T07:13:57,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731827637734, max timestamp=1731827637783Num uniques 1086; 2024-11-17T07:13:57,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-17T07:13:57,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-17T07:13:57,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731827637734, max timestamp=1731827637783Num uniques 1086; 2024-11-17T07:13:57,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-17T07:13:57,794 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/122ff82f2d5c4b5cb651bffa13378cd0, entries=100, sequenceid=598, filesize=9.6 K 2024-11-17T07:13:57,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/78c0e65443214dfc9c15dc3d36ab7f22 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/78c0e65443214dfc9c15dc3d36ab7f22 2024-11-17T07:13:57,805 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/78c0e65443214dfc9c15dc3d36ab7f22, entries=100, sequenceid=598, filesize=9.6 K 2024-11-17T07:13:57,807 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=53.67 KB/54960 for cccbc0d9c2f0bc3781663c8c9bb7223f in 552ms, sequenceid=598, compaction requested=false 2024-11-17T07:13:57,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:57,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:57,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-17T07:13:57,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-17T07:13:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:57,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=60.38 KB heapSize=158.63 KB 2024-11-17T07:13:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:57,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:57,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/bf7e135b2521467a9b21c2a3b79d1b4b is 50, key is test_row_0/A:col10/1731827637790/Put/seqid=0 2024-11-17T07:13:57,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-17T07:13:57,816 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 867 msec 2024-11-17T07:13:57,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 880 msec 2024-11-17T07:13:57,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741933_1109 (size=14741) 2024-11-17T07:13:57,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=677 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/bf7e135b2521467a9b21c2a3b79d1b4b 2024-11-17T07:13:57,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/ae8b578fad804236bc3e5459470145ed is 50, key is test_row_0/B:col10/1731827637790/Put/seqid=0 2024-11-17T07:13:57,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741934_1110 (size=12301) 2024-11-17T07:13:57,941 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827637912, max timestamp=1731827637938Num uniques 859; 2024-11-17T07:13:57,942 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:13:57,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827637912, max timestamp=1731827637938Num uniques 859; 2024-11-17T07:13:57,945 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:13:57,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827637912, max timestamp=1731827637938Num uniques 859; 2024-11-17T07:13:57,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:13:58,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:58,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 341 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827698049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:58,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:58,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-17T07:13:58,079 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:13:58,079 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:13:58,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-17T07:13:58,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T07:13:58,082 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:13:58,083 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:13:58,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:13:58,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:58,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 342 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827698167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:58,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54838 deadline: 1731827698177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:58,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54886 deadline: 1731827698179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:58,183 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cccbc0d9c2f0bc3781663c8c9bb7223f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:13:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:58,184 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cccbc0d9c2f0bc3781663c8c9bb7223f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:13:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:58,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T07:13:58,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:58,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54852 deadline: 1731827698187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:58,191 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cccbc0d9c2f0bc3781663c8c9bb7223f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:13:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:58,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:58,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54882 deadline: 1731827698207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:58,210 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cccbc0d9c2f0bc3781663c8c9bb7223f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:13:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:58,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T07:13:58,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:58,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:58,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:58,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:58,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:58,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=677 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/ae8b578fad804236bc3e5459470145ed 2024-11-17T07:13:58,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/dac679695e0c485fa84118fee545f532 is 50, key is test_row_0/C:col10/1731827637790/Put/seqid=0 2024-11-17T07:13:58,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741935_1111 (size=12301) 2024-11-17T07:13:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 343 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827698377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:58,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T07:13:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:58,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:58,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T07:13:58,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T07:13:58,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:58,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:58,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:58,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:58,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:58,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:58,686 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=677 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/dac679695e0c485fa84118fee545f532 2024-11-17T07:13:58,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:58,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 344 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827698687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:58,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T07:13:58,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:58,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:58,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:58,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:58,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:58,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:13:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T07:13:58,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/bf7e135b2521467a9b21c2a3b79d1b4b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/bf7e135b2521467a9b21c2a3b79d1b4b 2024-11-17T07:13:58,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/bf7e135b2521467a9b21c2a3b79d1b4b, entries=200, sequenceid=677, filesize=14.4 K 2024-11-17T07:13:58,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/ae8b578fad804236bc3e5459470145ed as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/ae8b578fad804236bc3e5459470145ed 2024-11-17T07:13:58,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/ae8b578fad804236bc3e5459470145ed, entries=150, sequenceid=677, filesize=12.0 K 2024-11-17T07:13:58,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/dac679695e0c485fa84118fee545f532 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/dac679695e0c485fa84118fee545f532 2024-11-17T07:13:58,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/dac679695e0c485fa84118fee545f532, entries=150, sequenceid=677, filesize=12.0 K 2024-11-17T07:13:58,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.58 KB/162384, currentSize=127.47 KB/130530 for cccbc0d9c2f0bc3781663c8c9bb7223f in 932ms, sequenceid=677, compaction requested=true 2024-11-17T07:13:58,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:58,742 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:58,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:13:58,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:58,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:13:58,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:58,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:13:58,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:58,743 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:58,743 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37921 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:58,744 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/A is initiating minor compaction (all files) 2024-11-17T07:13:58,744 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/A in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:58,744 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7727fdb9c3f147f6b7a07a50c2bab19d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ad7eb9502a8d48e2964d5735f85b4907, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/bf7e135b2521467a9b21c2a3b79d1b4b] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=37.0 K 2024-11-17T07:13:58,745 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7727fdb9c3f147f6b7a07a50c2bab19d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731827636015 2024-11-17T07:13:58,745 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:58,745 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/B is initiating minor compaction (all files) 2024-11-17T07:13:58,745 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/B in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:58,745 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/5fb20b956cf1412dbd53a25ca908db84, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/122ff82f2d5c4b5cb651bffa13378cd0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/ae8b578fad804236bc3e5459470145ed] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=34.6 K 2024-11-17T07:13:58,746 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad7eb9502a8d48e2964d5735f85b4907, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=598, earliestPutTs=1731827636698 2024-11-17T07:13:58,749 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fb20b956cf1412dbd53a25ca908db84, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731827636015 2024-11-17T07:13:58,749 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 122ff82f2d5c4b5cb651bffa13378cd0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=598, earliestPutTs=1731827636698 2024-11-17T07:13:58,752 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf7e135b2521467a9b21c2a3b79d1b4b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=677, earliestPutTs=1731827637762 2024-11-17T07:13:58,752 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting ae8b578fad804236bc3e5459470145ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=677, earliestPutTs=1731827637790 2024-11-17T07:13:58,779 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:58,780 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/ee19e18e29c94dafa50ff9d03d206dbc is 50, key is test_row_0/A:col10/1731827637790/Put/seqid=0 2024-11-17T07:13:58,785 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:58,786 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/8a0ad09b11af45da90f035d2759a2d94 is 50, key is test_row_0/B:col10/1731827637790/Put/seqid=0 2024-11-17T07:13:58,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741936_1112 (size=13425) 2024-11-17T07:13:58,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741937_1113 (size=13425) 2024-11-17T07:13:58,828 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/8a0ad09b11af45da90f035d2759a2d94 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/8a0ad09b11af45da90f035d2759a2d94 2024-11-17T07:13:58,832 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/ee19e18e29c94dafa50ff9d03d206dbc as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ee19e18e29c94dafa50ff9d03d206dbc 2024-11-17T07:13:58,838 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/B of cccbc0d9c2f0bc3781663c8c9bb7223f into 8a0ad09b11af45da90f035d2759a2d94(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:58,838 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:58,838 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/B, priority=13, startTime=1731827638743; duration=0sec 2024-11-17T07:13:58,838 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:13:58,838 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:B 2024-11-17T07:13:58,838 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:13:58,839 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:13:58,840 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/C is initiating minor compaction (all files) 2024-11-17T07:13:58,840 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/C in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:58,840 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/13524d4862ce40389fc3996b288bf79e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/78c0e65443214dfc9c15dc3d36ab7f22, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/dac679695e0c485fa84118fee545f532] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=34.6 K 2024-11-17T07:13:58,840 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/A of cccbc0d9c2f0bc3781663c8c9bb7223f into ee19e18e29c94dafa50ff9d03d206dbc(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:58,840 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 13524d4862ce40389fc3996b288bf79e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=590, earliestPutTs=1731827636015 2024-11-17T07:13:58,840 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:58,841 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/A, priority=13, startTime=1731827638742; duration=0sec 2024-11-17T07:13:58,841 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:58,841 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:A 2024-11-17T07:13:58,841 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 78c0e65443214dfc9c15dc3d36ab7f22, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=598, earliestPutTs=1731827636698 2024-11-17T07:13:58,841 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting dac679695e0c485fa84118fee545f532, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=677, earliestPutTs=1731827637790 2024-11-17T07:13:58,855 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#C#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:13:58,856 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/432d5817404e4d4aac1148e41fd0b8b6 is 50, key is test_row_0/C:col10/1731827637790/Put/seqid=0 2024-11-17T07:13:58,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-17T07:13:58,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:13:58,859 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-17T07:13:58,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:13:58,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:58,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:13:58,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:58,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:13:58,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:13:58,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741938_1114 (size=13425) 2024-11-17T07:13:58,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/cf98d6d0fdf74dfebf94ef091f87cdf2 is 50, key is test_row_0/A:col10/1731827638031/Put/seqid=0 2024-11-17T07:13:58,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741939_1115 (size=12301) 2024-11-17T07:13:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:13:59,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:13:59,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T07:13:59,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:59,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 353 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827699248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:59,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:59,276 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/432d5817404e4d4aac1148e41fd0b8b6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/432d5817404e4d4aac1148e41fd0b8b6 2024-11-17T07:13:59,284 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/C of cccbc0d9c2f0bc3781663c8c9bb7223f into 432d5817404e4d4aac1148e41fd0b8b6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:13:59,285 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:13:59,285 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/C, priority=13, startTime=1731827638743; duration=0sec 2024-11-17T07:13:59,285 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:13:59,285 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:C 2024-11-17T07:13:59,302 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/cf98d6d0fdf74dfebf94ef091f87cdf2 2024-11-17T07:13:59,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/bfd50bcbcc034128a1bb0830a58b918e is 50, key is test_row_0/B:col10/1731827638031/Put/seqid=0 2024-11-17T07:13:59,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741940_1116 (size=12301) 2024-11-17T07:13:59,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 354 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827699358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:59,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:59,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:59,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:59,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 355 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827699567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:59,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:59,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:59,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:13:59,735 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/bfd50bcbcc034128a1bb0830a58b918e 2024-11-17T07:13:59,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/2ef35ddf83c24901806e313f8f1ae536 is 50, key is test_row_0/C:col10/1731827638031/Put/seqid=0 2024-11-17T07:13:59,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741941_1117 (size=12301) 2024-11-17T07:13:59,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:13:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 356 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827699877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:13:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:13:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:00,162 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/2ef35ddf83c24901806e313f8f1ae536 2024-11-17T07:14:00,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/cf98d6d0fdf74dfebf94ef091f87cdf2 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cf98d6d0fdf74dfebf94ef091f87cdf2 2024-11-17T07:14:00,175 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cf98d6d0fdf74dfebf94ef091f87cdf2, entries=150, sequenceid=714, filesize=12.0 K 2024-11-17T07:14:00,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/bfd50bcbcc034128a1bb0830a58b918e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/bfd50bcbcc034128a1bb0830a58b918e 2024-11-17T07:14:00,187 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/bfd50bcbcc034128a1bb0830a58b918e, entries=150, sequenceid=714, filesize=12.0 K 2024-11-17T07:14:00,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/2ef35ddf83c24901806e313f8f1ae536 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2ef35ddf83c24901806e313f8f1ae536 2024-11-17T07:14:00,196 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2ef35ddf83c24901806e313f8f1ae536, entries=150, sequenceid=714, filesize=12.0 K 2024-11-17T07:14:00,197 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for cccbc0d9c2f0bc3781663c8c9bb7223f in 1338ms, sequenceid=714, compaction requested=false 2024-11-17T07:14:00,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:14:00,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:14:00,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-17T07:14:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-17T07:14:00,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-17T07:14:00,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1160 sec 2024-11-17T07:14:00,203 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.1230 sec 2024-11-17T07:14:00,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-17T07:14:00,237 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:00,238 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:00,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-17T07:14:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T07:14:00,241 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:00,242 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:00,242 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:00,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T07:14:00,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:14:00,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T07:14:00,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:14:00,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:00,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:14:00,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:00,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:14:00,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:00,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T07:14:00,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:14:00,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:14:00,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:14:00,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:00,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:00,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/152c43fd242c4b42b72093885c9d1bb7 is 50, key is test_row_0/A:col10/1731827639216/Put/seqid=0 2024-11-17T07:14:00,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741942_1118 (size=14741) 2024-11-17T07:14:00,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/152c43fd242c4b42b72093885c9d1bb7 2024-11-17T07:14:00,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/38e282aa4cc14dbeb6d554505c48e411 is 50, key is test_row_0/B:col10/1731827639216/Put/seqid=0 2024-11-17T07:14:00,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741943_1119 (size=12301) 2024-11-17T07:14:00,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/38e282aa4cc14dbeb6d554505c48e411 2024-11-17T07:14:00,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/2eca7ae921cd470290a0042d203cc5f5 is 50, key is test_row_0/C:col10/1731827639216/Put/seqid=0 2024-11-17T07:14:00,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741944_1120 (size=12301) 2024-11-17T07:14:00,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=727 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/2eca7ae921cd470290a0042d203cc5f5 2024-11-17T07:14:00,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 375 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827700482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:00,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:00,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/152c43fd242c4b42b72093885c9d1bb7 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/152c43fd242c4b42b72093885c9d1bb7 2024-11-17T07:14:00,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/152c43fd242c4b42b72093885c9d1bb7, entries=200, sequenceid=727, filesize=14.4 K 2024-11-17T07:14:00,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/38e282aa4cc14dbeb6d554505c48e411 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/38e282aa4cc14dbeb6d554505c48e411 2024-11-17T07:14:00,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/38e282aa4cc14dbeb6d554505c48e411, entries=150, sequenceid=727, filesize=12.0 K 2024-11-17T07:14:00,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/2eca7ae921cd470290a0042d203cc5f5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2eca7ae921cd470290a0042d203cc5f5 2024-11-17T07:14:00,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2eca7ae921cd470290a0042d203cc5f5, entries=150, sequenceid=727, filesize=12.0 K 2024-11-17T07:14:00,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cccbc0d9c2f0bc3781663c8c9bb7223f in 124ms, sequenceid=727, compaction requested=true 2024-11-17T07:14:00,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:14:00,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:00,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:00,514 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:00,514 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:00,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:00,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:00,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:00,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:00,515 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:00,515 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/B is initiating minor compaction (all files) 2024-11-17T07:14:00,516 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/B in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:14:00,516 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/8a0ad09b11af45da90f035d2759a2d94, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/bfd50bcbcc034128a1bb0830a58b918e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/38e282aa4cc14dbeb6d554505c48e411] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=37.1 K 2024-11-17T07:14:00,516 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:00,516 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/A is initiating minor compaction (all files) 2024-11-17T07:14:00,516 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/A in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:14:00,516 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ee19e18e29c94dafa50ff9d03d206dbc, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cf98d6d0fdf74dfebf94ef091f87cdf2, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/152c43fd242c4b42b72093885c9d1bb7] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=39.5 K 2024-11-17T07:14:00,516 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a0ad09b11af45da90f035d2759a2d94, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=677, earliestPutTs=1731827637790 2024-11-17T07:14:00,517 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee19e18e29c94dafa50ff9d03d206dbc, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=677, earliestPutTs=1731827637790 2024-11-17T07:14:00,517 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting bfd50bcbcc034128a1bb0830a58b918e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731827638031 2024-11-17T07:14:00,518 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf98d6d0fdf74dfebf94ef091f87cdf2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731827638031 2024-11-17T07:14:00,518 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 38e282aa4cc14dbeb6d554505c48e411, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1731827639216 2024-11-17T07:14:00,519 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 152c43fd242c4b42b72093885c9d1bb7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1731827639216 2024-11-17T07:14:00,548 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#B#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:00,549 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/143d243981e94b2ea38db4d18ecf2f0c is 50, key is test_row_0/B:col10/1731827639216/Put/seqid=0 2024-11-17T07:14:00,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-17T07:14:00,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:14:00,550 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T07:14:00,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:14:00,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:00,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:14:00,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:00,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:14:00,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T07:14:00,571 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#A#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:00,572 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/d5235df9ee794d1ba5748bdab2997355 is 50, key is test_row_0/A:col10/1731827639216/Put/seqid=0 2024-11-17T07:14:00,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/f4eaab5cda634cba90cc6200f6b20117 is 50, key is test_row_0/A:col10/1731827640478/Put/seqid=0 2024-11-17T07:14:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:14:00,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. as already flushing 2024-11-17T07:14:00,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741945_1121 (size=13527) 2024-11-17T07:14:00,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741946_1122 (size=13527) 2024-11-17T07:14:00,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 385 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827700655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:00,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741947_1123 (size=12301) 2024-11-17T07:14:00,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 386 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827700767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T07:14:00,973 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-17T07:14:00,973 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:00,976 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-17T07:14:00,976 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:00,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:00,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 387 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827700977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:00,977 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-17T07:14:00,978 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:00,978 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-17T07:14:00,978 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:00,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:00,980 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:00,980 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:00,980 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:00,980 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:00,981 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:00,981 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:00,981 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:00,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:00,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:00,981 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:00,981 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:00,982 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:01,028 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/143d243981e94b2ea38db4d18ecf2f0c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/143d243981e94b2ea38db4d18ecf2f0c 2024-11-17T07:14:01,035 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/B of cccbc0d9c2f0bc3781663c8c9bb7223f into 143d243981e94b2ea38db4d18ecf2f0c(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:01,035 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:14:01,035 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/B, priority=13, startTime=1731827640514; duration=0sec 2024-11-17T07:14:01,035 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:01,035 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:B 2024-11-17T07:14:01,035 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:01,037 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:01,037 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cccbc0d9c2f0bc3781663c8c9bb7223f/C is initiating minor compaction (all files) 2024-11-17T07:14:01,037 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cccbc0d9c2f0bc3781663c8c9bb7223f/C in TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:14:01,037 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/432d5817404e4d4aac1148e41fd0b8b6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2ef35ddf83c24901806e313f8f1ae536, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2eca7ae921cd470290a0042d203cc5f5] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp, totalSize=37.1 K 2024-11-17T07:14:01,038 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 432d5817404e4d4aac1148e41fd0b8b6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=677, earliestPutTs=1731827637790 2024-11-17T07:14:01,038 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ef35ddf83c24901806e313f8f1ae536, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731827638031 2024-11-17T07:14:01,038 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2eca7ae921cd470290a0042d203cc5f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=727, earliestPutTs=1731827639216 2024-11-17T07:14:01,047 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cccbc0d9c2f0bc3781663c8c9bb7223f#C#compaction#111 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:01,048 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/b7bb1ac20b134cb38562b3d85b0600a9 is 50, key is test_row_0/C:col10/1731827639216/Put/seqid=0 2024-11-17T07:14:01,052 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/d5235df9ee794d1ba5748bdab2997355 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d5235df9ee794d1ba5748bdab2997355 2024-11-17T07:14:01,060 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/A of cccbc0d9c2f0bc3781663c8c9bb7223f into d5235df9ee794d1ba5748bdab2997355(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:01,060 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:14:01,060 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/A, priority=13, startTime=1731827640514; duration=0sec 2024-11-17T07:14:01,060 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:01,061 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:A 2024-11-17T07:14:01,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741948_1124 (size=13527) 2024-11-17T07:14:01,079 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/f4eaab5cda634cba90cc6200f6b20117 2024-11-17T07:14:01,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/2281573ad172418aa1f921691705ecff is 50, key is test_row_0/B:col10/1731827640478/Put/seqid=0 2024-11-17T07:14:01,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741949_1125 (size=12301) 2024-11-17T07:14:01,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:01,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 388 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827701287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:01,288 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T07:14:01,470 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/b7bb1ac20b134cb38562b3d85b0600a9 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/b7bb1ac20b134cb38562b3d85b0600a9 2024-11-17T07:14:01,476 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cccbc0d9c2f0bc3781663c8c9bb7223f/C of cccbc0d9c2f0bc3781663c8c9bb7223f into b7bb1ac20b134cb38562b3d85b0600a9(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:01,476 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:14:01,476 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., storeName=cccbc0d9c2f0bc3781663c8c9bb7223f/C, priority=13, startTime=1731827640514; duration=0sec 2024-11-17T07:14:01,476 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:01,476 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:C 2024-11-17T07:14:01,495 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/2281573ad172418aa1f921691705ecff 2024-11-17T07:14:01,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/077b6ef112a04bbeb98c75dec3a4e1b1 is 50, key is test_row_0/C:col10/1731827640478/Put/seqid=0 2024-11-17T07:14:01,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741950_1126 (size=12301) 2024-11-17T07:14:01,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:01,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 389 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54914 deadline: 1731827701797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:01,798 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:01,910 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=747 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/077b6ef112a04bbeb98c75dec3a4e1b1 2024-11-17T07:14:01,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/f4eaab5cda634cba90cc6200f6b20117 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f4eaab5cda634cba90cc6200f6b20117 2024-11-17T07:14:01,919 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f4eaab5cda634cba90cc6200f6b20117, entries=150, sequenceid=747, filesize=12.0 K 2024-11-17T07:14:01,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/2281573ad172418aa1f921691705ecff as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/2281573ad172418aa1f921691705ecff 2024-11-17T07:14:01,924 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/2281573ad172418aa1f921691705ecff, entries=150, sequenceid=747, filesize=12.0 K 2024-11-17T07:14:01,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/077b6ef112a04bbeb98c75dec3a4e1b1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/077b6ef112a04bbeb98c75dec3a4e1b1 2024-11-17T07:14:01,929 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/077b6ef112a04bbeb98c75dec3a4e1b1, entries=150, sequenceid=747, filesize=12.0 K 2024-11-17T07:14:01,930 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for cccbc0d9c2f0bc3781663c8c9bb7223f in 1380ms, sequenceid=747, compaction requested=false 2024-11-17T07:14:01,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:14:01,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:14:01,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-17T07:14:01,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-17T07:14:01,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-17T07:14:01,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6890 sec 2024-11-17T07:14:01,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.6950 sec 2024-11-17T07:14:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:14:02,198 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T07:14:02,198 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-17T07:14:02,198 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:02,198 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:02,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:14:02,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:02,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:14:02,199 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:02,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:02,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:02,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:14:02,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:02,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/7f49673b85a74bd593c9a114933832d5 is 50, key is test_row_0/A:col10/1731827640651/Put/seqid=0 2024-11-17T07:14:02,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741951_1127 (size=12301) 2024-11-17T07:14:02,229 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-17T07:14:02,229 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:02,229 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:02,229 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:02,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:02,229 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-17T07:14:02,229 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-17T07:14:02,229 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:02,229 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:02,229 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:02,229 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:02,229 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:02,229 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:02,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:02,229 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-17T07:14:02,388 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:02,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/7f49673b85a74bd593c9a114933832d5 2024-11-17T07:14:02,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/f0f685c17b5c4f3c8a280a17e0b84a2a is 50, key is test_row_0/B:col10/1731827640651/Put/seqid=0 2024-11-17T07:14:02,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741952_1128 (size=12301) 2024-11-17T07:14:02,809 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-17T07:14:02,809 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:02,809 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:02,809 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:02,809 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:02,809 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T07:14:02,810 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 328 2024-11-17T07:14:02,810 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-17T07:14:02,810 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-17T07:14:02,810 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-17T07:14:02,810 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-17T07:14:02,810 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T07:14:02,810 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2548 2024-11-17T07:14:02,810 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2361 2024-11-17T07:14:02,810 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T07:14:02,810 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1104 2024-11-17T07:14:02,810 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3309 rows 2024-11-17T07:14:02,810 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1102 2024-11-17T07:14:02,810 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3299 rows 2024-11-17T07:14:02,810 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T07:14:02,810 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T07:14:02,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:02,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:02,810 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:02,814 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T07:14:02,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:02,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T07:14:02,821 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827642821"}]},"ts":"1731827642821"} 2024-11-17T07:14:02,824 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T07:14:02,824 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T07:14:02,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T07:14:02,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cccbc0d9c2f0bc3781663c8c9bb7223f, UNASSIGN}] 2024-11-17T07:14:02,830 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cccbc0d9c2f0bc3781663c8c9bb7223f, UNASSIGN 2024-11-17T07:14:02,831 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=cccbc0d9c2f0bc3781663c8c9bb7223f, regionState=CLOSING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:02,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cccbc0d9c2f0bc3781663c8c9bb7223f, UNASSIGN because future has completed 2024-11-17T07:14:02,834 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T07:14:02,834 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:14:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T07:14:02,931 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T07:14:02,989 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:14:02,989 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T07:14:02,990 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing cccbc0d9c2f0bc3781663c8c9bb7223f, disabling compactions & flushes 2024-11-17T07:14:02,990 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:14:03,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/f0f685c17b5c4f3c8a280a17e0b84a2a 2024-11-17T07:14:03,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/a193b016558845f6a726c5d560d171a0 is 50, key is test_row_0/C:col10/1731827640651/Put/seqid=0 2024-11-17T07:14:03,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741953_1129 (size=12301) 2024-11-17T07:14:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T07:14:03,432 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=763 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/a193b016558845f6a726c5d560d171a0 2024-11-17T07:14:03,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/7f49673b85a74bd593c9a114933832d5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7f49673b85a74bd593c9a114933832d5 2024-11-17T07:14:03,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7f49673b85a74bd593c9a114933832d5, entries=150, sequenceid=763, filesize=12.0 K 2024-11-17T07:14:03,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/f0f685c17b5c4f3c8a280a17e0b84a2a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/f0f685c17b5c4f3c8a280a17e0b84a2a 2024-11-17T07:14:03,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/f0f685c17b5c4f3c8a280a17e0b84a2a, entries=150, sequenceid=763, filesize=12.0 K 2024-11-17T07:14:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T07:14:03,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/a193b016558845f6a726c5d560d171a0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/a193b016558845f6a726c5d560d171a0 2024-11-17T07:14:03,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/a193b016558845f6a726c5d560d171a0, entries=150, sequenceid=763, filesize=12.0 K 2024-11-17T07:14:03,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=26.84 KB/27480 for cccbc0d9c2f0bc3781663c8c9bb7223f in 1254ms, sequenceid=763, compaction requested=true 2024-11-17T07:14:03,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cccbc0d9c2f0bc3781663c8c9bb7223f: 2024-11-17T07:14:03,452 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:14:03,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:03,452 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:14:03,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:03,452 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. after waiting 0 ms 2024-11-17T07:14:03,453 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:14:03,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:03,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:03,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cccbc0d9c2f0bc3781663c8c9bb7223f:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:03,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:03,453 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. because compaction request was cancelled 2024-11-17T07:14:03,453 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing cccbc0d9c2f0bc3781663c8c9bb7223f 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T07:14:03,453 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:A 2024-11-17T07:14:03,453 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. because compaction request was cancelled 2024-11-17T07:14:03,453 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. because compaction request was cancelled 2024-11-17T07:14:03,453 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:C 2024-11-17T07:14:03,453 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cccbc0d9c2f0bc3781663c8c9bb7223f:B 2024-11-17T07:14:03,453 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=A 2024-11-17T07:14:03,453 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:03,453 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=B 2024-11-17T07:14:03,453 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:03,453 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cccbc0d9c2f0bc3781663c8c9bb7223f, store=C 2024-11-17T07:14:03,453 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:03,456 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/014d585450ff4c0694b8068d41b342fa is 50, key is test_row_2/A:col10/1731827642808/Put/seqid=0 2024-11-17T07:14:03,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741954_1130 (size=7415) 2024-11-17T07:14:03,861 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=770 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/014d585450ff4c0694b8068d41b342fa 2024-11-17T07:14:03,868 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/bfd43bf86f664fb892ac4f26f5752ad6 is 50, key is test_row_2/B:col10/1731827642808/Put/seqid=0 2024-11-17T07:14:03,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741955_1131 (size=7415) 2024-11-17T07:14:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T07:14:04,273 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=770 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/bfd43bf86f664fb892ac4f26f5752ad6 2024-11-17T07:14:04,281 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/ccf79a7c184a4ecc88f01e3d372157a6 is 50, key is test_row_2/C:col10/1731827642808/Put/seqid=0 2024-11-17T07:14:04,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741956_1132 (size=7415) 2024-11-17T07:14:04,685 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=770 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/ccf79a7c184a4ecc88f01e3d372157a6 2024-11-17T07:14:04,690 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/A/014d585450ff4c0694b8068d41b342fa as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/014d585450ff4c0694b8068d41b342fa 2024-11-17T07:14:04,694 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/014d585450ff4c0694b8068d41b342fa, entries=50, sequenceid=770, filesize=7.2 K 2024-11-17T07:14:04,695 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/B/bfd43bf86f664fb892ac4f26f5752ad6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/bfd43bf86f664fb892ac4f26f5752ad6 2024-11-17T07:14:04,698 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/bfd43bf86f664fb892ac4f26f5752ad6, entries=50, sequenceid=770, filesize=7.2 K 2024-11-17T07:14:04,699 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/.tmp/C/ccf79a7c184a4ecc88f01e3d372157a6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/ccf79a7c184a4ecc88f01e3d372157a6 2024-11-17T07:14:04,703 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/ccf79a7c184a4ecc88f01e3d372157a6, entries=50, sequenceid=770, filesize=7.2 K 2024-11-17T07:14:04,704 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for cccbc0d9c2f0bc3781663c8c9bb7223f in 1251ms, sequenceid=770, compaction requested=true 2024-11-17T07:14:04,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5034bfd5dbf149c99152ec1d61f67794, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/abb64e39ddf54b0b953da0718c77aed5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f2b64651920b442c9249258210a7ffdc, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/603373856f174b35b023d3788736e1f3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7ec4ad4ac3be4f33a4e3f6ac60bd5a52, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a66e896b67e64a2bb67f50a92b57d1cc, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/bd1b53ad0e8b42d49ac7ff8beec281d3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d38357fbb7064a00a936e311bfa91ea8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/21e399f9e561430685df1b5bd42ea8c3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d5ee7059807b453cb8410eb1808e6510, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5fa82aeb03db421db486e8c1c8e564d7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/20e47e2a01ad4cd6b2ac6056bb92db70, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/4eb6f4b9787b45f4bcd78e29022e26a4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/020e3eb62169417c82e4e1e375e4345b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/b833a96f35aa452ca57d3fc76a292f04, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/c1bd9ff043e34fd09df31240e3c92b85, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a6e832f36f494a29bbc121901e3e8c12, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a32f055771c4406c83de9c8c3e60b71b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/baaf67a34abf4558b4001fb4ccfa29f3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/567cba1c294f4923ac4c30a94d05bcae, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5657e2e8a8c94520872c2ada618e3ec9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cbbb4db557f54cc4ad533d7c01c96f09, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ab31d3c3042040dfb80c60f990c46623, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f66384ea5fda4e1f9625a44cd53d518b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/aaa31809583a449981cf390cb0cd9f77, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cf48b7519fd548beabf7885bd64a559a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/23ab2450ccd5402f806e55a7b07a91ee, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f2c7a4814b634775bddf0a3697a47f4f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/8005d19361fb4ecf8127b161006b1e3d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/80ce2c406c38409c8ad517c1e6a2c1a6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7727fdb9c3f147f6b7a07a50c2bab19d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ad7eb9502a8d48e2964d5735f85b4907, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/bf7e135b2521467a9b21c2a3b79d1b4b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ee19e18e29c94dafa50ff9d03d206dbc, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cf98d6d0fdf74dfebf94ef091f87cdf2, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/152c43fd242c4b42b72093885c9d1bb7] to archive 2024-11-17T07:14:04,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:14:04,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5034bfd5dbf149c99152ec1d61f67794 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5034bfd5dbf149c99152ec1d61f67794 2024-11-17T07:14:04,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/abb64e39ddf54b0b953da0718c77aed5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/abb64e39ddf54b0b953da0718c77aed5 2024-11-17T07:14:04,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f2b64651920b442c9249258210a7ffdc to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f2b64651920b442c9249258210a7ffdc 2024-11-17T07:14:04,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/603373856f174b35b023d3788736e1f3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/603373856f174b35b023d3788736e1f3 2024-11-17T07:14:04,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7ec4ad4ac3be4f33a4e3f6ac60bd5a52 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7ec4ad4ac3be4f33a4e3f6ac60bd5a52 2024-11-17T07:14:04,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a66e896b67e64a2bb67f50a92b57d1cc to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a66e896b67e64a2bb67f50a92b57d1cc 2024-11-17T07:14:04,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/bd1b53ad0e8b42d49ac7ff8beec281d3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/bd1b53ad0e8b42d49ac7ff8beec281d3 2024-11-17T07:14:04,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d38357fbb7064a00a936e311bfa91ea8 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d38357fbb7064a00a936e311bfa91ea8 2024-11-17T07:14:04,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/21e399f9e561430685df1b5bd42ea8c3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/21e399f9e561430685df1b5bd42ea8c3 2024-11-17T07:14:04,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d5ee7059807b453cb8410eb1808e6510 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d5ee7059807b453cb8410eb1808e6510 2024-11-17T07:14:04,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5fa82aeb03db421db486e8c1c8e564d7 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5fa82aeb03db421db486e8c1c8e564d7 2024-11-17T07:14:04,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/20e47e2a01ad4cd6b2ac6056bb92db70 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/20e47e2a01ad4cd6b2ac6056bb92db70 2024-11-17T07:14:04,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/4eb6f4b9787b45f4bcd78e29022e26a4 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/4eb6f4b9787b45f4bcd78e29022e26a4 2024-11-17T07:14:04,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/020e3eb62169417c82e4e1e375e4345b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/020e3eb62169417c82e4e1e375e4345b 2024-11-17T07:14:04,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/b833a96f35aa452ca57d3fc76a292f04 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/b833a96f35aa452ca57d3fc76a292f04 2024-11-17T07:14:04,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/c1bd9ff043e34fd09df31240e3c92b85 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/c1bd9ff043e34fd09df31240e3c92b85 2024-11-17T07:14:04,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a6e832f36f494a29bbc121901e3e8c12 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a6e832f36f494a29bbc121901e3e8c12 2024-11-17T07:14:04,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a32f055771c4406c83de9c8c3e60b71b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/a32f055771c4406c83de9c8c3e60b71b 2024-11-17T07:14:04,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/baaf67a34abf4558b4001fb4ccfa29f3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/baaf67a34abf4558b4001fb4ccfa29f3 2024-11-17T07:14:04,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/567cba1c294f4923ac4c30a94d05bcae to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/567cba1c294f4923ac4c30a94d05bcae 2024-11-17T07:14:04,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5657e2e8a8c94520872c2ada618e3ec9 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/5657e2e8a8c94520872c2ada618e3ec9 2024-11-17T07:14:04,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cbbb4db557f54cc4ad533d7c01c96f09 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cbbb4db557f54cc4ad533d7c01c96f09 2024-11-17T07:14:04,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ab31d3c3042040dfb80c60f990c46623 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ab31d3c3042040dfb80c60f990c46623 2024-11-17T07:14:04,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f66384ea5fda4e1f9625a44cd53d518b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f66384ea5fda4e1f9625a44cd53d518b 2024-11-17T07:14:04,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/aaa31809583a449981cf390cb0cd9f77 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/aaa31809583a449981cf390cb0cd9f77 2024-11-17T07:14:04,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cf48b7519fd548beabf7885bd64a559a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cf48b7519fd548beabf7885bd64a559a 2024-11-17T07:14:04,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/23ab2450ccd5402f806e55a7b07a91ee to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/23ab2450ccd5402f806e55a7b07a91ee 2024-11-17T07:14:04,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f2c7a4814b634775bddf0a3697a47f4f to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f2c7a4814b634775bddf0a3697a47f4f 2024-11-17T07:14:04,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/8005d19361fb4ecf8127b161006b1e3d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/8005d19361fb4ecf8127b161006b1e3d 2024-11-17T07:14:04,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/80ce2c406c38409c8ad517c1e6a2c1a6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/80ce2c406c38409c8ad517c1e6a2c1a6 2024-11-17T07:14:04,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7727fdb9c3f147f6b7a07a50c2bab19d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7727fdb9c3f147f6b7a07a50c2bab19d 2024-11-17T07:14:04,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ad7eb9502a8d48e2964d5735f85b4907 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ad7eb9502a8d48e2964d5735f85b4907 2024-11-17T07:14:04,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/bf7e135b2521467a9b21c2a3b79d1b4b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/bf7e135b2521467a9b21c2a3b79d1b4b 2024-11-17T07:14:04,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ee19e18e29c94dafa50ff9d03d206dbc to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/ee19e18e29c94dafa50ff9d03d206dbc 2024-11-17T07:14:04,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cf98d6d0fdf74dfebf94ef091f87cdf2 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/cf98d6d0fdf74dfebf94ef091f87cdf2 2024-11-17T07:14:04,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/152c43fd242c4b42b72093885c9d1bb7 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/152c43fd242c4b42b72093885c9d1bb7 2024-11-17T07:14:04,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/15f61ef21ae44a04bae58f481563367a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7cacf3c77f6e42eabba31f3777cf264e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/a9cc6888f6724ab7981789b6e3211505, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/96fb862a18b44ccea403db0d0ef64542, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/5b479f11f68e489b8fde70aa061dcda6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/2d1195ccc6fd4c199355d06bb6fc0a76, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d5f3033be5d3427da198bbbf2c13b462, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/276634b39b144d8eb784ba8789701ba4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/142ff8a9b5d84f98937f3fa59d34cd53, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/9cf63011940643e7919c9b67b08edfcd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/71a33c9f65b94a52920ee5dacfa1761e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/53aefa13f7b0494b96e100867d1aebe8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/8ce49fb80da2445d9fbf3b5a31b48bc7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/4103d81e897840dfa681ddaa5771bd47, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/28cc6a096e8f4264b4a226a1e32bbd0a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/f6f2771ced354b88bc050db42db67e52, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/49856b44e4fe4abab7e1b205f0604798, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d543037de57f4261b35895d3e9247bd1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/88a065ba61404bc18250c3f939fe4bf3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d26f1f705cc747278f7a53206b50d539, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7a5a187f502e47a191a2be95d25acf84, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/e01c41b5b8954c82a58e4e134e9f7f54, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/52cd311ae2324ff6ab791dc4b0783f6a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/9d627176b551498eb3ffbff8b50388ee, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/1375cd3adb034d8c9a4e26df6ef79b53, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/268e9ea3ee96402e81b1e8d306c2a730, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/fe0998c5ed04454da2c8f327f3ee97b3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/839842ba29754f7691637ba1de8cc902, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/294edc772665413fa2578676b0332a54, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/5fb20b956cf1412dbd53a25ca908db84, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7f7cf1bdb66845ed99c0070ba01ac7de, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/122ff82f2d5c4b5cb651bffa13378cd0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/8a0ad09b11af45da90f035d2759a2d94, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/ae8b578fad804236bc3e5459470145ed, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/bfd50bcbcc034128a1bb0830a58b918e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/38e282aa4cc14dbeb6d554505c48e411] to archive 2024-11-17T07:14:04,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:14:04,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/15f61ef21ae44a04bae58f481563367a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/15f61ef21ae44a04bae58f481563367a 2024-11-17T07:14:04,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7cacf3c77f6e42eabba31f3777cf264e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7cacf3c77f6e42eabba31f3777cf264e 2024-11-17T07:14:04,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/a9cc6888f6724ab7981789b6e3211505 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/a9cc6888f6724ab7981789b6e3211505 2024-11-17T07:14:04,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/96fb862a18b44ccea403db0d0ef64542 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/96fb862a18b44ccea403db0d0ef64542 2024-11-17T07:14:04,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/5b479f11f68e489b8fde70aa061dcda6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/5b479f11f68e489b8fde70aa061dcda6 2024-11-17T07:14:04,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/2d1195ccc6fd4c199355d06bb6fc0a76 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/2d1195ccc6fd4c199355d06bb6fc0a76 2024-11-17T07:14:04,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d5f3033be5d3427da198bbbf2c13b462 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d5f3033be5d3427da198bbbf2c13b462 2024-11-17T07:14:04,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/276634b39b144d8eb784ba8789701ba4 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/276634b39b144d8eb784ba8789701ba4 2024-11-17T07:14:04,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/142ff8a9b5d84f98937f3fa59d34cd53 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/142ff8a9b5d84f98937f3fa59d34cd53 2024-11-17T07:14:04,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/9cf63011940643e7919c9b67b08edfcd to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/9cf63011940643e7919c9b67b08edfcd 2024-11-17T07:14:04,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/71a33c9f65b94a52920ee5dacfa1761e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/71a33c9f65b94a52920ee5dacfa1761e 2024-11-17T07:14:04,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/53aefa13f7b0494b96e100867d1aebe8 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/53aefa13f7b0494b96e100867d1aebe8 2024-11-17T07:14:04,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/8ce49fb80da2445d9fbf3b5a31b48bc7 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/8ce49fb80da2445d9fbf3b5a31b48bc7 2024-11-17T07:14:04,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/4103d81e897840dfa681ddaa5771bd47 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/4103d81e897840dfa681ddaa5771bd47 2024-11-17T07:14:04,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/28cc6a096e8f4264b4a226a1e32bbd0a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/28cc6a096e8f4264b4a226a1e32bbd0a 2024-11-17T07:14:04,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/f6f2771ced354b88bc050db42db67e52 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/f6f2771ced354b88bc050db42db67e52 2024-11-17T07:14:04,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/49856b44e4fe4abab7e1b205f0604798 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/49856b44e4fe4abab7e1b205f0604798 2024-11-17T07:14:04,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d543037de57f4261b35895d3e9247bd1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d543037de57f4261b35895d3e9247bd1 2024-11-17T07:14:04,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/88a065ba61404bc18250c3f939fe4bf3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/88a065ba61404bc18250c3f939fe4bf3 2024-11-17T07:14:04,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d26f1f705cc747278f7a53206b50d539 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/d26f1f705cc747278f7a53206b50d539 2024-11-17T07:14:04,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7a5a187f502e47a191a2be95d25acf84 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7a5a187f502e47a191a2be95d25acf84 2024-11-17T07:14:04,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/e01c41b5b8954c82a58e4e134e9f7f54 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/e01c41b5b8954c82a58e4e134e9f7f54 2024-11-17T07:14:04,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/52cd311ae2324ff6ab791dc4b0783f6a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/52cd311ae2324ff6ab791dc4b0783f6a 2024-11-17T07:14:04,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/9d627176b551498eb3ffbff8b50388ee to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/9d627176b551498eb3ffbff8b50388ee 2024-11-17T07:14:04,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/1375cd3adb034d8c9a4e26df6ef79b53 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/1375cd3adb034d8c9a4e26df6ef79b53 2024-11-17T07:14:04,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/268e9ea3ee96402e81b1e8d306c2a730 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/268e9ea3ee96402e81b1e8d306c2a730 2024-11-17T07:14:04,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/fe0998c5ed04454da2c8f327f3ee97b3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/fe0998c5ed04454da2c8f327f3ee97b3 2024-11-17T07:14:04,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/839842ba29754f7691637ba1de8cc902 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/839842ba29754f7691637ba1de8cc902 2024-11-17T07:14:04,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/294edc772665413fa2578676b0332a54 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/294edc772665413fa2578676b0332a54 2024-11-17T07:14:04,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/5fb20b956cf1412dbd53a25ca908db84 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/5fb20b956cf1412dbd53a25ca908db84 2024-11-17T07:14:04,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7f7cf1bdb66845ed99c0070ba01ac7de to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/7f7cf1bdb66845ed99c0070ba01ac7de 2024-11-17T07:14:04,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/122ff82f2d5c4b5cb651bffa13378cd0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/122ff82f2d5c4b5cb651bffa13378cd0 2024-11-17T07:14:04,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/8a0ad09b11af45da90f035d2759a2d94 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/8a0ad09b11af45da90f035d2759a2d94 2024-11-17T07:14:04,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/ae8b578fad804236bc3e5459470145ed to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/ae8b578fad804236bc3e5459470145ed 2024-11-17T07:14:04,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/bfd50bcbcc034128a1bb0830a58b918e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/bfd50bcbcc034128a1bb0830a58b918e 2024-11-17T07:14:04,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/38e282aa4cc14dbeb6d554505c48e411 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/38e282aa4cc14dbeb6d554505c48e411 2024-11-17T07:14:04,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/18216dfea1bd42aba8fb5d4f07f4d94f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/880c637fd9b14b6cbd5abee6f89e053a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d452c1c7307f4481a8cb08cbd4fc747a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/56191a8cf672443bb5a0cde616a69456, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d9d9b493102344439feaed5995c38cbc, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d4dc5400ab0e4f3c94d6741de24632f8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/1c51a8c20b1c48a4bc46a3120ab519a4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9b975ca33ff742a08871a3935f657deb, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d5fa87b02b6f4419b0f165f350f99d5c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d65104a3164947d7ad8ee361880ac158, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/29d25cc0ac88485bb0bcc01b22ae1a74, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/7b4bcbc3b1404684bd9daaa5c2cc1e2c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2eea551a2db2492c931e0a1c5a947885, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/14b39cb6dba6491a96ff132b7847af0d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9a33e72adc3f4ea191099ce704cf8418, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/78ebaeddc73b4a03986ed067088a1561, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/0c39f90a90c2437cb71f56dfa587b671, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/3eb5014dd9634a319f79cafa921266d7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9235a4bfd3c045d7a028ff0cde01a5f8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/afde37dfb9524673a1877ac1448dc3a3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9fbd4084f71b43dcbc9d40f7498811a6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/afddaf9aa48140aeb851621566fdfbcf, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/782f87993fb146c99d32a04c16624511, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/87590ac893f94b30a83b4aa8855c3109, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/6d6bbc7731c245f0b3e18ffc9f4823a1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/eaee706b17d540f88f31eeebaa4928e5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/04dacb77c83847f48e3b6d7e5127bfc1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/30797e3f6415468fabf4122996c3b84c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/465853729bdf46b88f4b3423d0ecf346, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/13524d4862ce40389fc3996b288bf79e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/03cd2c51e9ab41adbdf0c5daaea3c9a6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/78c0e65443214dfc9c15dc3d36ab7f22, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/432d5817404e4d4aac1148e41fd0b8b6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/dac679695e0c485fa84118fee545f532, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2ef35ddf83c24901806e313f8f1ae536, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2eca7ae921cd470290a0042d203cc5f5] to archive 2024-11-17T07:14:04,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:14:04,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/18216dfea1bd42aba8fb5d4f07f4d94f to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/18216dfea1bd42aba8fb5d4f07f4d94f 2024-11-17T07:14:04,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/880c637fd9b14b6cbd5abee6f89e053a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/880c637fd9b14b6cbd5abee6f89e053a 2024-11-17T07:14:04,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d452c1c7307f4481a8cb08cbd4fc747a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d452c1c7307f4481a8cb08cbd4fc747a 2024-11-17T07:14:04,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/56191a8cf672443bb5a0cde616a69456 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/56191a8cf672443bb5a0cde616a69456 2024-11-17T07:14:04,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d9d9b493102344439feaed5995c38cbc to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d9d9b493102344439feaed5995c38cbc 2024-11-17T07:14:04,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d4dc5400ab0e4f3c94d6741de24632f8 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d4dc5400ab0e4f3c94d6741de24632f8 2024-11-17T07:14:04,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/1c51a8c20b1c48a4bc46a3120ab519a4 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/1c51a8c20b1c48a4bc46a3120ab519a4 2024-11-17T07:14:04,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9b975ca33ff742a08871a3935f657deb to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9b975ca33ff742a08871a3935f657deb 2024-11-17T07:14:04,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d5fa87b02b6f4419b0f165f350f99d5c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d5fa87b02b6f4419b0f165f350f99d5c 2024-11-17T07:14:04,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d65104a3164947d7ad8ee361880ac158 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/d65104a3164947d7ad8ee361880ac158 2024-11-17T07:14:04,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/29d25cc0ac88485bb0bcc01b22ae1a74 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/29d25cc0ac88485bb0bcc01b22ae1a74 2024-11-17T07:14:04,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/7b4bcbc3b1404684bd9daaa5c2cc1e2c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/7b4bcbc3b1404684bd9daaa5c2cc1e2c 2024-11-17T07:14:04,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2eea551a2db2492c931e0a1c5a947885 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2eea551a2db2492c931e0a1c5a947885 2024-11-17T07:14:04,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/14b39cb6dba6491a96ff132b7847af0d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/14b39cb6dba6491a96ff132b7847af0d 2024-11-17T07:14:04,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9a33e72adc3f4ea191099ce704cf8418 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9a33e72adc3f4ea191099ce704cf8418 2024-11-17T07:14:04,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/78ebaeddc73b4a03986ed067088a1561 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/78ebaeddc73b4a03986ed067088a1561 2024-11-17T07:14:04,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/0c39f90a90c2437cb71f56dfa587b671 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/0c39f90a90c2437cb71f56dfa587b671 2024-11-17T07:14:04,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/3eb5014dd9634a319f79cafa921266d7 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/3eb5014dd9634a319f79cafa921266d7 2024-11-17T07:14:04,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9235a4bfd3c045d7a028ff0cde01a5f8 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9235a4bfd3c045d7a028ff0cde01a5f8 2024-11-17T07:14:04,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/afde37dfb9524673a1877ac1448dc3a3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/afde37dfb9524673a1877ac1448dc3a3 2024-11-17T07:14:04,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9fbd4084f71b43dcbc9d40f7498811a6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/9fbd4084f71b43dcbc9d40f7498811a6 2024-11-17T07:14:04,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/afddaf9aa48140aeb851621566fdfbcf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/afddaf9aa48140aeb851621566fdfbcf 2024-11-17T07:14:04,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/782f87993fb146c99d32a04c16624511 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/782f87993fb146c99d32a04c16624511 2024-11-17T07:14:04,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/87590ac893f94b30a83b4aa8855c3109 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/87590ac893f94b30a83b4aa8855c3109 2024-11-17T07:14:04,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/6d6bbc7731c245f0b3e18ffc9f4823a1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/6d6bbc7731c245f0b3e18ffc9f4823a1 2024-11-17T07:14:04,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/eaee706b17d540f88f31eeebaa4928e5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/eaee706b17d540f88f31eeebaa4928e5 2024-11-17T07:14:04,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/04dacb77c83847f48e3b6d7e5127bfc1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/04dacb77c83847f48e3b6d7e5127bfc1 2024-11-17T07:14:04,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/30797e3f6415468fabf4122996c3b84c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/30797e3f6415468fabf4122996c3b84c 2024-11-17T07:14:04,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/465853729bdf46b88f4b3423d0ecf346 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/465853729bdf46b88f4b3423d0ecf346 2024-11-17T07:14:04,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/13524d4862ce40389fc3996b288bf79e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/13524d4862ce40389fc3996b288bf79e 2024-11-17T07:14:04,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/03cd2c51e9ab41adbdf0c5daaea3c9a6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/03cd2c51e9ab41adbdf0c5daaea3c9a6 2024-11-17T07:14:04,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/78c0e65443214dfc9c15dc3d36ab7f22 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/78c0e65443214dfc9c15dc3d36ab7f22 2024-11-17T07:14:04,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/432d5817404e4d4aac1148e41fd0b8b6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/432d5817404e4d4aac1148e41fd0b8b6 2024-11-17T07:14:04,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/dac679695e0c485fa84118fee545f532 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/dac679695e0c485fa84118fee545f532 2024-11-17T07:14:04,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2ef35ddf83c24901806e313f8f1ae536 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2ef35ddf83c24901806e313f8f1ae536 2024-11-17T07:14:04,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2eca7ae921cd470290a0042d203cc5f5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/2eca7ae921cd470290a0042d203cc5f5 2024-11-17T07:14:04,857 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/recovered.edits/773.seqid, newMaxSeqId=773, maxSeqId=1 2024-11-17T07:14:04,858 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. 2024-11-17T07:14:04,858 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for cccbc0d9c2f0bc3781663c8c9bb7223f: Waiting for close lock at 1731827642990Running coprocessor pre-close hooks at 1731827642990Disabling compacts and flushes for region at 1731827642990Disabling writes for close at 1731827643453 (+463 ms)Obtaining lock to block concurrent updates at 1731827643453Preparing flush snapshotting stores in cccbc0d9c2f0bc3781663c8c9bb7223f at 1731827643453Finished memstore snapshotting TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731827643453Flushing stores of TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f. at 1731827643453Flushing cccbc0d9c2f0bc3781663c8c9bb7223f/A: creating writer at 1731827643454 (+1 ms)Flushing cccbc0d9c2f0bc3781663c8c9bb7223f/A: appending metadata at 1731827643456 (+2 ms)Flushing cccbc0d9c2f0bc3781663c8c9bb7223f/A: closing flushed file at 1731827643456Flushing cccbc0d9c2f0bc3781663c8c9bb7223f/B: creating writer at 1731827643866 (+410 ms)Flushing cccbc0d9c2f0bc3781663c8c9bb7223f/B: appending metadata at 1731827643868 (+2 ms)Flushing cccbc0d9c2f0bc3781663c8c9bb7223f/B: closing flushed file at 1731827643868Flushing cccbc0d9c2f0bc3781663c8c9bb7223f/C: creating writer at 1731827644278 (+410 ms)Flushing cccbc0d9c2f0bc3781663c8c9bb7223f/C: appending metadata at 1731827644280 (+2 ms)Flushing cccbc0d9c2f0bc3781663c8c9bb7223f/C: closing flushed file at 1731827644280Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c56bcbf: reopening flushed file at 1731827644689 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c1010c0: reopening flushed file at 1731827644694 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@713c4a27: reopening flushed file at 1731827644698 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for cccbc0d9c2f0bc3781663c8c9bb7223f in 1251ms, sequenceid=770, compaction requested=true at 1731827644704 (+6 ms)Writing region close event to WAL at 1731827644853 (+149 ms)Running coprocessor post-close hooks at 1731827644858 (+5 ms)Closed at 1731827644858 2024-11-17T07:14:04,861 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:14:04,861 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=cccbc0d9c2f0bc3781663c8c9bb7223f, regionState=CLOSED 2024-11-17T07:14:04,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:14:04,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-17T07:14:04,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure cccbc0d9c2f0bc3781663c8c9bb7223f, server=1dd62e24b303,36419,1731827614816 in 2.0300 sec 2024-11-17T07:14:04,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-17T07:14:04,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cccbc0d9c2f0bc3781663c8c9bb7223f, UNASSIGN in 2.0370 sec 2024-11-17T07:14:04,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-17T07:14:04,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0440 sec 2024-11-17T07:14:04,872 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827644872"}]},"ts":"1731827644872"} 2024-11-17T07:14:04,874 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T07:14:04,874 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T07:14:04,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0590 sec 2024-11-17T07:14:04,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-17T07:14:04,978 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:04,981 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T07:14:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:04,986 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:04,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T07:14:04,987 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:04,992 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:14:04,995 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/recovered.edits] 2024-11-17T07:14:04,998 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/014d585450ff4c0694b8068d41b342fa to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/014d585450ff4c0694b8068d41b342fa 2024-11-17T07:14:05,000 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7f49673b85a74bd593c9a114933832d5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/7f49673b85a74bd593c9a114933832d5 2024-11-17T07:14:05,001 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d5235df9ee794d1ba5748bdab2997355 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/d5235df9ee794d1ba5748bdab2997355 2024-11-17T07:14:05,002 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f4eaab5cda634cba90cc6200f6b20117 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/A/f4eaab5cda634cba90cc6200f6b20117 2024-11-17T07:14:05,005 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/143d243981e94b2ea38db4d18ecf2f0c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/143d243981e94b2ea38db4d18ecf2f0c 2024-11-17T07:14:05,006 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/2281573ad172418aa1f921691705ecff to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/2281573ad172418aa1f921691705ecff 2024-11-17T07:14:05,007 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/bfd43bf86f664fb892ac4f26f5752ad6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/bfd43bf86f664fb892ac4f26f5752ad6 2024-11-17T07:14:05,008 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/f0f685c17b5c4f3c8a280a17e0b84a2a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/B/f0f685c17b5c4f3c8a280a17e0b84a2a 2024-11-17T07:14:05,011 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/077b6ef112a04bbeb98c75dec3a4e1b1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/077b6ef112a04bbeb98c75dec3a4e1b1 2024-11-17T07:14:05,012 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/a193b016558845f6a726c5d560d171a0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/a193b016558845f6a726c5d560d171a0 2024-11-17T07:14:05,014 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/b7bb1ac20b134cb38562b3d85b0600a9 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/b7bb1ac20b134cb38562b3d85b0600a9 2024-11-17T07:14:05,015 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/ccf79a7c184a4ecc88f01e3d372157a6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/C/ccf79a7c184a4ecc88f01e3d372157a6 2024-11-17T07:14:05,017 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/recovered.edits/773.seqid to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f/recovered.edits/773.seqid 2024-11-17T07:14:05,018 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cccbc0d9c2f0bc3781663c8c9bb7223f 2024-11-17T07:14:05,018 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T07:14:05,023 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:05,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-17T07:14:05,034 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T07:14:05,067 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T07:14:05,068 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:05,068 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T07:14:05,069 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731827645068"}]},"ts":"9223372036854775807"} 2024-11-17T07:14:05,072 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T07:14:05,072 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cccbc0d9c2f0bc3781663c8c9bb7223f, NAME => 'TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T07:14:05,072 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T07:14:05,073 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731827645072"}]},"ts":"9223372036854775807"} 2024-11-17T07:14:05,075 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T07:14:05,075 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:05,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 94 msec 2024-11-17T07:14:05,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-17T07:14:05,098 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T07:14:05,098 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:05,108 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_411634984_22 at /127.0.0.1:33740 [Waiting for operation #311] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-49537162_22 at /127.0.0.1:46208 [Waiting for operation #129] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;1dd62e24b303:36419-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-49537162_22 at /127.0.0.1:45732 [Waiting for operation #299] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=419 (was 258) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2242 (was 2049) - AvailableMemoryMB LEAK? - 2024-11-17T07:14:05,117 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=419, ProcessCount=11, AvailableMemoryMB=2242 2024-11-17T07:14:05,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T07:14:05,118 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T07:14:05,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:05,120 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T07:14:05,120 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:05,121 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-17T07:14:05,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-17T07:14:05,121 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T07:14:05,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741957_1133 (size=1061) 2024-11-17T07:14:05,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-17T07:14:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-17T07:14:05,529 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650 2024-11-17T07:14:05,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741958_1134 (size=53) 2024-11-17T07:14:05,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-17T07:14:05,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:14:05,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing efcef908f9da2d14e10a65888eb3eb6d, disabling compactions & flushes 2024-11-17T07:14:05,935 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:05,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:05,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. after waiting 0 ms 2024-11-17T07:14:05,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:05,935 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:05,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for efcef908f9da2d14e10a65888eb3eb6d: Waiting for close lock at 1731827645935Disabling compacts and flushes for region at 1731827645935Disabling writes for close at 1731827645935Writing region close event to WAL at 1731827645935Closed at 1731827645935 2024-11-17T07:14:05,936 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T07:14:05,936 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731827645936"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731827645936"}]},"ts":"1731827645936"} 2024-11-17T07:14:05,938 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T07:14:05,939 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T07:14:05,939 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827645939"}]},"ts":"1731827645939"} 2024-11-17T07:14:05,941 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T07:14:05,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, ASSIGN}] 2024-11-17T07:14:05,942 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, ASSIGN 2024-11-17T07:14:05,943 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, ASSIGN; state=OFFLINE, location=1dd62e24b303,36419,1731827614816; forceNewPlan=false, retain=false 2024-11-17T07:14:06,093 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=efcef908f9da2d14e10a65888eb3eb6d, regionState=OPENING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:06,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, ASSIGN because future has completed 2024-11-17T07:14:06,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:14:06,251 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:06,251 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} 2024-11-17T07:14:06,251 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:06,251 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:14:06,251 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:06,251 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:06,252 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:06,254 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:14:06,254 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efcef908f9da2d14e10a65888eb3eb6d columnFamilyName A 2024-11-17T07:14:06,255 DEBUG [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:06,255 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.HStore(327): Store=efcef908f9da2d14e10a65888eb3eb6d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:14:06,255 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:06,256 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:14:06,256 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efcef908f9da2d14e10a65888eb3eb6d columnFamilyName B 2024-11-17T07:14:06,256 DEBUG [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:06,257 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.HStore(327): Store=efcef908f9da2d14e10a65888eb3eb6d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:14:06,257 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:06,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-17T07:14:06,258 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:14:06,258 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efcef908f9da2d14e10a65888eb3eb6d columnFamilyName C 2024-11-17T07:14:06,258 DEBUG [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:06,258 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.HStore(327): Store=efcef908f9da2d14e10a65888eb3eb6d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:14:06,258 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:06,259 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:06,259 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:06,259 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:06,260 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:06,260 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:06,261 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T07:14:06,262 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:06,264 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T07:14:06,264 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened efcef908f9da2d14e10a65888eb3eb6d; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74904963, jitterRate=0.11617092788219452}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T07:14:06,264 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:06,265 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for efcef908f9da2d14e10a65888eb3eb6d: Running coprocessor pre-open hook at 1731827646251Writing region info on filesystem at 1731827646251Initializing all the Stores at 1731827646252 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827646252Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827646252Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827646252Cleaning up temporary data from old regions at 1731827646260 (+8 ms)Running coprocessor post-open hooks at 1731827646264 (+4 ms)Region opened successfully at 1731827646265 (+1 ms) 2024-11-17T07:14:06,266 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., pid=32, masterSystemTime=1731827646247 2024-11-17T07:14:06,268 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:06,268 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:06,268 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=efcef908f9da2d14e10a65888eb3eb6d, regionState=OPEN, openSeqNum=2, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:06,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:14:06,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-17T07:14:06,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 in 175 msec 2024-11-17T07:14:06,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-17T07:14:06,275 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, ASSIGN in 332 msec 2024-11-17T07:14:06,275 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T07:14:06,276 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827646275"}]},"ts":"1731827646275"} 2024-11-17T07:14:06,277 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T07:14:06,278 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T07:14:06,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1600 sec 2024-11-17T07:14:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-17T07:14:07,268 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:07,269 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a00013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:07,269 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:07,269 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:07,271 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e595508, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:07,273 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:07,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e686d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:07,274 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:07,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:07,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:07,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:07,278 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T07:14:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 1dd62e24b303,36491,1731827613877 2024-11-17T07:14:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@29fa5185 2024-11-17T07:14:07,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T07:14:07,279 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T07:14:07,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T07:14:07,284 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T07:14:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:07,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741959_1135 (size=1097) 2024-11-17T07:14:07,702 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T07:14:07,702 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T07:14:07,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T07:14:07,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, REOPEN/MOVE}] 2024-11-17T07:14:07,712 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, REOPEN/MOVE 2024-11-17T07:14:07,713 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=efcef908f9da2d14e10a65888eb3eb6d, regionState=CLOSING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:07,715 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, REOPEN/MOVE because future has completed 2024-11-17T07:14:07,715 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T07:14:07,715 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:14:07,868 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:07,868 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T07:14:07,868 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing efcef908f9da2d14e10a65888eb3eb6d, disabling compactions & flushes 2024-11-17T07:14:07,868 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:07,868 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:07,868 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. after waiting 0 ms 2024-11-17T07:14:07,868 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:07,872 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T07:14:07,872 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:07,872 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for efcef908f9da2d14e10a65888eb3eb6d: Waiting for close lock at 1731827647868Running coprocessor pre-close hooks at 1731827647868Disabling compacts and flushes for region at 1731827647868Disabling writes for close at 1731827647868Writing region close event to WAL at 1731827647868Running coprocessor post-close hooks at 1731827647872 (+4 ms)Closed at 1731827647872 2024-11-17T07:14:07,872 WARN [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: efcef908f9da2d14e10a65888eb3eb6d to self. 2024-11-17T07:14:07,874 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:07,875 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=efcef908f9da2d14e10a65888eb3eb6d, regionState=CLOSED 2024-11-17T07:14:07,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:14:07,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-17T07:14:07,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 in 162 msec 2024-11-17T07:14:07,879 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, REOPEN/MOVE; state=CLOSED, location=1dd62e24b303,36419,1731827614816; forceNewPlan=false, retain=true 2024-11-17T07:14:08,030 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=efcef908f9da2d14e10a65888eb3eb6d, regionState=OPENING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, REOPEN/MOVE because future has completed 2024-11-17T07:14:08,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:14:08,187 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:08,187 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} 2024-11-17T07:14:08,188 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,188 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:14:08,188 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,188 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,193 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,193 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:14:08,198 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efcef908f9da2d14e10a65888eb3eb6d columnFamilyName A 2024-11-17T07:14:08,200 DEBUG [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:08,200 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.HStore(327): Store=efcef908f9da2d14e10a65888eb3eb6d/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:14:08,200 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,201 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:14:08,201 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efcef908f9da2d14e10a65888eb3eb6d columnFamilyName B 2024-11-17T07:14:08,201 DEBUG [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:08,202 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.HStore(327): Store=efcef908f9da2d14e10a65888eb3eb6d/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:14:08,202 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,202 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:14:08,202 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region efcef908f9da2d14e10a65888eb3eb6d columnFamilyName C 2024-11-17T07:14:08,202 DEBUG [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:08,203 INFO [StoreOpener-efcef908f9da2d14e10a65888eb3eb6d-1 {}] regionserver.HStore(327): Store=efcef908f9da2d14e10a65888eb3eb6d/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:14:08,203 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:08,203 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,204 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,205 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,206 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,206 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,206 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T07:14:08,208 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,209 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened efcef908f9da2d14e10a65888eb3eb6d; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61331921, jitterRate=-0.08608315885066986}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T07:14:08,209 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,210 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for efcef908f9da2d14e10a65888eb3eb6d: Running coprocessor pre-open hook at 1731827648188Writing region info on filesystem at 1731827648188Initializing all the Stores at 1731827648189 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827648189Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827648189Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827648189Cleaning up temporary data from old regions at 1731827648206 (+17 ms)Running coprocessor post-open hooks at 1731827648209 (+3 ms)Region opened successfully at 1731827648210 (+1 ms) 2024-11-17T07:14:08,211 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., pid=37, masterSystemTime=1731827648184 2024-11-17T07:14:08,213 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:08,213 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:08,214 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=efcef908f9da2d14e10a65888eb3eb6d, regionState=OPEN, openSeqNum=5, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:14:08,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-17T07:14:08,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 in 184 msec 2024-11-17T07:14:08,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-17T07:14:08,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, REOPEN/MOVE in 506 msec 2024-11-17T07:14:08,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-17T07:14:08,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 516 msec 2024-11-17T07:14:08,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-17T07:14:08,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-17T07:14:08,230 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:08,231 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad8b864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,231 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:08,231 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:08,232 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:08,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10feb5ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:08,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:08,234 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:08,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180c6c34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,236 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:08,236 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:08,237 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@273813dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:08,237 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:08,238 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:08,238 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7788b4f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,240 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,240 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:08,240 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:08,241 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:08,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f155ad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:08,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:08,243 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:08,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b64e766, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:08,244 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:08,245 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:08,245 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c34e189, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:08,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:08,247 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:08,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3759d7ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,248 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:08,248 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:08,249 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dbc19c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:08,251 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38832, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c149c71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,252 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,252 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:08,252 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:08,253 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f915ddb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:08,254 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54d78908, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:08,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:08,256 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@293659ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:08,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:08,257 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169788b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:08,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:08,259 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3764eaed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:08,260 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:08,261 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e62cbdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:08,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:08,262 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ea8846, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:08,262 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:08,263 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:08,263 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d35809d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:08,264 DEBUG [Thread-682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:08,264 DEBUG [Thread-679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:08,264 DEBUG [Thread-677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:08,264 DEBUG [Thread-671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:08,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:08,266 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:08,267 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:08,267 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:08,267 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:08,267 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:08,267 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:08,267 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:08,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-17T07:14:08,270 DEBUG [Thread-675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:08,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:08,271 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:08,268 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:08,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:08,271 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:08,271 DEBUG [Thread-673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:08,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:08,272 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:08,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-17T07:14:08,273 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:14:08,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:14:08,274 DEBUG [Thread-673 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:08,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:14:08,276 DEBUG [Thread-673 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:08,278 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:14:08,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53120, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:14:08,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:14:08,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:14:08,280 DEBUG [Thread-688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:08,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:14:08,282 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:14:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:14:08,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:08,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:08,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:08,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:08,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:08,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:08,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175db2fcbc77904faeabaf5cbe5c421b41_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827648275/Put/seqid=0 2024-11-17T07:14:08,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741960_1136 (size=12154) 2024-11-17T07:14:08,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-17T07:14:08,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827708347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731827648326, max timestamp=1731827648362Num uniques 859; 2024-11-17T07:14:08,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1731827648326, max timestamp=1731827648362Num uniques 859; 2024-11-17T07:14:08,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731827648326, max timestamp=1731827648362Num uniques 859; 2024-11-17T07:14:08,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:14:08,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-17T07:14:08,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-17T07:14:08,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T07:14:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:08,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:08,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:08,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:08,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827708421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827708423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827708445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827708448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827708497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827708557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827708558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827708558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827708558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T07:14:08,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:08,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:08,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:08,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:08,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:08,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-17T07:14:08,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827708707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T07:14:08,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:08,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:08,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:08,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:08,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:08,755 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:08,761 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175db2fcbc77904faeabaf5cbe5c421b41_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175db2fcbc77904faeabaf5cbe5c421b41_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:08,762 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/6c7302d343a04535961ed4f04694b404, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:08,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827708767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827708767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827708767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:08,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827708768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:08,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:08,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/6c7302d343a04535961ed4f04694b404 is 175, key is test_row_0/A:col10/1731827648275/Put/seqid=0 2024-11-17T07:14:08,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741961_1137 (size=30955) 2024-11-17T07:14:08,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T07:14:08,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:08,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:08,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:08,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:08,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:08,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:08,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-17T07:14:09,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827709017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:09,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T07:14:09,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:09,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:09,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:09,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:09,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:09,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:09,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:09,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827709077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:09,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827709077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:09,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:09,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827709077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:09,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:09,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827709078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,086 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:09,186 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/6c7302d343a04535961ed4f04694b404 2024-11-17T07:14:09,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T07:14:09,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:09,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:09,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:09,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:09,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:09,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/2ce36032795e4f92836349496d87c9a9 is 50, key is test_row_0/B:col10/1731827648275/Put/seqid=0 2024-11-17T07:14:09,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741962_1138 (size=12001) 2024-11-17T07:14:09,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/2ce36032795e4f92836349496d87c9a9 2024-11-17T07:14:09,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/96baeb837c3047c789aa2b69b1525e69 is 50, key is test_row_0/C:col10/1731827648275/Put/seqid=0 2024-11-17T07:14:09,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741963_1139 (size=12001) 2024-11-17T07:14:09,270 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/96baeb837c3047c789aa2b69b1525e69 2024-11-17T07:14:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/6c7302d343a04535961ed4f04694b404 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/6c7302d343a04535961ed4f04694b404 2024-11-17T07:14:09,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/6c7302d343a04535961ed4f04694b404, entries=150, sequenceid=15, filesize=30.2 K 2024-11-17T07:14:09,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/2ce36032795e4f92836349496d87c9a9 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/2ce36032795e4f92836349496d87c9a9 2024-11-17T07:14:09,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/2ce36032795e4f92836349496d87c9a9, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T07:14:09,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/96baeb837c3047c789aa2b69b1525e69 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/96baeb837c3047c789aa2b69b1525e69 2024-11-17T07:14:09,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/96baeb837c3047c789aa2b69b1525e69, entries=150, sequenceid=15, filesize=11.7 K 2024-11-17T07:14:09,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=124.45 KB/127432 for efcef908f9da2d14e10a65888eb3eb6d in 1038ms, sequenceid=15, compaction requested=false 2024-11-17T07:14:09,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:09,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-17T07:14:09,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:09,357 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=124.45 KB heapSize=353.47 KB 2024-11-17T07:14:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:09,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:09,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174682e530b6df49179b730ce7bb69039a_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827648445/Put/seqid=0 2024-11-17T07:14:09,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741964_1140 (size=12154) 2024-11-17T07:14:09,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:09,403 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174682e530b6df49179b730ce7bb69039a_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174682e530b6df49179b730ce7bb69039a_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:09,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/e00872c4b3474f00b9e0e3dd87638bbf, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:09,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/e00872c4b3474f00b9e0e3dd87638bbf is 175, key is test_row_0/A:col10/1731827648445/Put/seqid=0 2024-11-17T07:14:09,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-17T07:14:09,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741965_1141 (size=30955) 2024-11-17T07:14:09,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:09,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:09,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827709563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:09,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827709588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:09,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827709597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:09,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827709597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827709597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:09,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827709677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:09,817 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/e00872c4b3474f00b9e0e3dd87638bbf 2024-11-17T07:14:09,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/c752fdfd6d5f414dbd3a54f9f7a652c4 is 50, key is test_row_0/B:col10/1731827648445/Put/seqid=0 2024-11-17T07:14:09,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741966_1142 (size=12001) 2024-11-17T07:14:09,842 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/c752fdfd6d5f414dbd3a54f9f7a652c4 2024-11-17T07:14:09,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/ad2fab4db0be44a9bcc6a87dbb12b0d8 is 50, key is test_row_0/C:col10/1731827648445/Put/seqid=0 2024-11-17T07:14:09,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741967_1143 (size=12001) 2024-11-17T07:14:09,878 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=39.86 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/ad2fab4db0be44a9bcc6a87dbb12b0d8 2024-11-17T07:14:09,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/e00872c4b3474f00b9e0e3dd87638bbf as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/e00872c4b3474f00b9e0e3dd87638bbf 2024-11-17T07:14:09,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:09,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827709888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:09,901 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/e00872c4b3474f00b9e0e3dd87638bbf, entries=150, sequenceid=50, filesize=30.2 K 2024-11-17T07:14:09,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/c752fdfd6d5f414dbd3a54f9f7a652c4 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/c752fdfd6d5f414dbd3a54f9f7a652c4 2024-11-17T07:14:09,911 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/c752fdfd6d5f414dbd3a54f9f7a652c4, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T07:14:09,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/ad2fab4db0be44a9bcc6a87dbb12b0d8 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/ad2fab4db0be44a9bcc6a87dbb12b0d8 2024-11-17T07:14:09,919 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/ad2fab4db0be44a9bcc6a87dbb12b0d8, entries=150, sequenceid=50, filesize=11.7 K 2024-11-17T07:14:09,920 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~124.45 KB/127432, heapSize ~353.42 KB/361904, currentSize=53.67 KB/54960 for efcef908f9da2d14e10a65888eb3eb6d in 563ms, sequenceid=50, compaction requested=false 2024-11-17T07:14:09,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:09,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:09,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-17T07:14:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-17T07:14:09,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-17T07:14:09,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6500 sec 2024-11-17T07:14:09,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.6590 sec 2024-11-17T07:14:10,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T07:14:10,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:10,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:10,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:10,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:10,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:10,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:10,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ef19b615ef23420996cfb2f56ac2eb3f_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827649558/Put/seqid=0 2024-11-17T07:14:10,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741968_1144 (size=14594) 2024-11-17T07:14:10,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827710299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:10,401 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T07:14:10,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827710407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-17T07:14:10,418 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:10,418 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-17T07:14:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T07:14:10,421 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:10,422 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:10,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T07:14:10,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T07:14:10,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:10,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:10,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:10,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:10,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827710607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:10,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827710607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:10,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827710607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:10,614 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in efcef908f9da2d14e10a65888eb3eb6d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:10,614 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827710607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:10,615 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in efcef908f9da2d14e10a65888eb3eb6d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,615 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in efcef908f9da2d14e10a65888eb3eb6d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:10,616 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in efcef908f9da2d14e10a65888eb3eb6d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:10,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827710617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:10,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:10,628 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:10,633 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ef19b615ef23420996cfb2f56ac2eb3f_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ef19b615ef23420996cfb2f56ac2eb3f_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:10,635 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/14be745c2a91483c9d5886d454a181ea, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:10,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/14be745c2a91483c9d5886d454a181ea is 175, key is test_row_0/A:col10/1731827649558/Put/seqid=0 2024-11-17T07:14:10,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741969_1145 (size=39549) 2024-11-17T07:14:10,648 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/14be745c2a91483c9d5886d454a181ea 2024-11-17T07:14:10,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/9a816cc616d8414fad92fdf0431b15c3 is 50, key is test_row_0/B:col10/1731827649558/Put/seqid=0 2024-11-17T07:14:10,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741970_1146 (size=12001) 2024-11-17T07:14:10,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/9a816cc616d8414fad92fdf0431b15c3 2024-11-17T07:14:10,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/a316ddaf691b4e338aa4db331a22ce9c is 50, key is test_row_0/C:col10/1731827649558/Put/seqid=0 2024-11-17T07:14:10,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741971_1147 (size=12001) 2024-11-17T07:14:10,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/a316ddaf691b4e338aa4db331a22ce9c 2024-11-17T07:14:10,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/14be745c2a91483c9d5886d454a181ea as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/14be745c2a91483c9d5886d454a181ea 2024-11-17T07:14:10,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/14be745c2a91483c9d5886d454a181ea, entries=200, sequenceid=62, filesize=38.6 K 2024-11-17T07:14:10,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/9a816cc616d8414fad92fdf0431b15c3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/9a816cc616d8414fad92fdf0431b15c3 2024-11-17T07:14:10,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/9a816cc616d8414fad92fdf0431b15c3, entries=150, sequenceid=62, filesize=11.7 K 2024-11-17T07:14:10,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/a316ddaf691b4e338aa4db331a22ce9c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/a316ddaf691b4e338aa4db331a22ce9c 2024-11-17T07:14:10,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/a316ddaf691b4e338aa4db331a22ce9c, entries=150, sequenceid=62, filesize=11.7 K 2024-11-17T07:14:10,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for efcef908f9da2d14e10a65888eb3eb6d in 531ms, sequenceid=62, compaction requested=true 2024-11-17T07:14:10,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:10,730 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:10,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-17T07:14:10,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:10,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:10,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:10,731 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T07:14:10,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:10,732 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:10,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:10,732 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/A is initiating minor compaction (all files) 2024-11-17T07:14:10,732 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/A in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:10,732 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/6c7302d343a04535961ed4f04694b404, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/e00872c4b3474f00b9e0e3dd87638bbf, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/14be745c2a91483c9d5886d454a181ea] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=99.1 K 2024-11-17T07:14:10,732 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:10,733 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/6c7302d343a04535961ed4f04694b404, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/e00872c4b3474f00b9e0e3dd87638bbf, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/14be745c2a91483c9d5886d454a181ea] 2024-11-17T07:14:10,733 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c7302d343a04535961ed4f04694b404, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731827648275 2024-11-17T07:14:10,734 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:10,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:10,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:10,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:10,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:10,734 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting e00872c4b3474f00b9e0e3dd87638bbf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731827648417 2024-11-17T07:14:10,736 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14be745c2a91483c9d5886d454a181ea, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731827649545 2024-11-17T07:14:10,736 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:10,736 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/B is initiating minor compaction (all files) 2024-11-17T07:14:10,736 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/B in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:10,736 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/2ce36032795e4f92836349496d87c9a9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/c752fdfd6d5f414dbd3a54f9f7a652c4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/9a816cc616d8414fad92fdf0431b15c3] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=35.2 K 2024-11-17T07:14:10,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T07:14:10,737 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ce36032795e4f92836349496d87c9a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731827648275 2024-11-17T07:14:10,739 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting c752fdfd6d5f414dbd3a54f9f7a652c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731827648417 2024-11-17T07:14:10,739 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a816cc616d8414fad92fdf0431b15c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731827649552 2024-11-17T07:14:10,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d938dea7b6e14297b1f1696a3d394193_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827650287/Put/seqid=0 2024-11-17T07:14:10,759 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#B#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:10,760 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/ff8fa00b9eb04bfba6c6302309b49194 is 50, key is test_row_0/B:col10/1731827649558/Put/seqid=0 2024-11-17T07:14:10,762 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:10,780 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117a3034a2940144167b1bc43fd19256751_efcef908f9da2d14e10a65888eb3eb6d store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:10,788 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117a3034a2940144167b1bc43fd19256751_efcef908f9da2d14e10a65888eb3eb6d, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:10,788 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a3034a2940144167b1bc43fd19256751_efcef908f9da2d14e10a65888eb3eb6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:10,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741972_1148 (size=12154) 2024-11-17T07:14:10,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:10,810 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d938dea7b6e14297b1f1696a3d394193_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d938dea7b6e14297b1f1696a3d394193_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:10,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/fb37c4659212467085a079f4ccbba955, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:10,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/fb37c4659212467085a079f4ccbba955 is 175, key is test_row_0/A:col10/1731827650287/Put/seqid=0 2024-11-17T07:14:10,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741973_1149 (size=12104) 2024-11-17T07:14:10,825 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/ff8fa00b9eb04bfba6c6302309b49194 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/ff8fa00b9eb04bfba6c6302309b49194 2024-11-17T07:14:10,832 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/B of efcef908f9da2d14e10a65888eb3eb6d into ff8fa00b9eb04bfba6c6302309b49194(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:10,832 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:10,832 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/B, priority=13, startTime=1731827650734; duration=0sec 2024-11-17T07:14:10,833 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:10,833 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:B 2024-11-17T07:14:10,834 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:10,837 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:10,837 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/C is initiating minor compaction (all files) 2024-11-17T07:14:10,837 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/C in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:10,838 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/96baeb837c3047c789aa2b69b1525e69, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/ad2fab4db0be44a9bcc6a87dbb12b0d8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/a316ddaf691b4e338aa4db331a22ce9c] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=35.2 K 2024-11-17T07:14:10,838 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 96baeb837c3047c789aa2b69b1525e69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731827648275 2024-11-17T07:14:10,839 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting ad2fab4db0be44a9bcc6a87dbb12b0d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731827648417 2024-11-17T07:14:10,839 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a316ddaf691b4e338aa4db331a22ce9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731827649552 2024-11-17T07:14:10,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741974_1150 (size=4469) 2024-11-17T07:14:10,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741975_1151 (size=30955) 2024-11-17T07:14:10,851 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:10,852 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/d3d73b550c7e4911b730e958f4a29e18 is 50, key is test_row_0/C:col10/1731827649558/Put/seqid=0 2024-11-17T07:14:10,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741976_1152 (size=12104) 2024-11-17T07:14:10,875 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/d3d73b550c7e4911b730e958f4a29e18 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d3d73b550c7e4911b730e958f4a29e18 2024-11-17T07:14:10,882 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/C of efcef908f9da2d14e10a65888eb3eb6d into d3d73b550c7e4911b730e958f4a29e18(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:10,882 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:10,882 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/C, priority=13, startTime=1731827650734; duration=0sec 2024-11-17T07:14:10,882 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:10,882 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:C 2024-11-17T07:14:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:10,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:10,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827710967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T07:14:11,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:11,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827711077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:11,081 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:11,246 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#A#compaction#131 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:11,247 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/a77efde900d541aabad20238bf4a3f09 is 175, key is test_row_0/A:col10/1731827649558/Put/seqid=0 2024-11-17T07:14:11,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741977_1153 (size=31058) 2024-11-17T07:14:11,251 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/fb37c4659212467085a079f4ccbba955 2024-11-17T07:14:11,258 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/a77efde900d541aabad20238bf4a3f09 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/a77efde900d541aabad20238bf4a3f09 2024-11-17T07:14:11,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/83f5157c9272425cba4ce373694db4b6 is 50, key is test_row_0/B:col10/1731827650287/Put/seqid=0 2024-11-17T07:14:11,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741978_1154 (size=12001) 2024-11-17T07:14:11,268 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/A of efcef908f9da2d14e10a65888eb3eb6d into a77efde900d541aabad20238bf4a3f09(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:11,268 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:11,268 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/83f5157c9272425cba4ce373694db4b6 2024-11-17T07:14:11,268 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/A, priority=13, startTime=1731827650730; duration=0sec 2024-11-17T07:14:11,268 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:11,268 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:A 2024-11-17T07:14:11,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/d0dfd12e8897460e88719ae7be2058b9 is 50, key is test_row_0/C:col10/1731827650287/Put/seqid=0 2024-11-17T07:14:11,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741979_1155 (size=12001) 2024-11-17T07:14:11,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827711287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:11,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:11,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:11,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T07:14:11,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827711597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:11,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:11,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:11,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:11,681 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/d0dfd12e8897460e88719ae7be2058b9 2024-11-17T07:14:11,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/fb37c4659212467085a079f4ccbba955 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/fb37c4659212467085a079f4ccbba955 2024-11-17T07:14:11,692 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/fb37c4659212467085a079f4ccbba955, entries=150, sequenceid=82, filesize=30.2 K 2024-11-17T07:14:11,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/83f5157c9272425cba4ce373694db4b6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/83f5157c9272425cba4ce373694db4b6 2024-11-17T07:14:11,697 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/83f5157c9272425cba4ce373694db4b6, entries=150, sequenceid=82, filesize=11.7 K 2024-11-17T07:14:11,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/d0dfd12e8897460e88719ae7be2058b9 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d0dfd12e8897460e88719ae7be2058b9 2024-11-17T07:14:11,703 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d0dfd12e8897460e88719ae7be2058b9, entries=150, sequenceid=82, filesize=11.7 K 2024-11-17T07:14:11,704 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for efcef908f9da2d14e10a65888eb3eb6d in 973ms, sequenceid=82, compaction requested=false 2024-11-17T07:14:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:11,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-17T07:14:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-17T07:14:11,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-17T07:14:11,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2860 sec 2024-11-17T07:14:11,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.2940 sec 2024-11-17T07:14:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:12,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T07:14:12,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:12,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:12,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:12,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:12,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:12,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:12,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bb2938e072824191bcdf2b579215dfad_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827650965/Put/seqid=0 2024-11-17T07:14:12,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741980_1156 (size=14594) 2024-11-17T07:14:12,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827712177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827712287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827712497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,500 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,523 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:12,529 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117bb2938e072824191bcdf2b579215dfad_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bb2938e072824191bcdf2b579215dfad_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:12,530 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/892bc61d5f6e40958f56f19d4f956c0a, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:12,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/892bc61d5f6e40958f56f19d4f956c0a is 175, key is test_row_0/A:col10/1731827650965/Put/seqid=0 2024-11-17T07:14:12,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741981_1157 (size=39549) 2024-11-17T07:14:12,538 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/892bc61d5f6e40958f56f19d4f956c0a 2024-11-17T07:14:12,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/f320de309ab547cc9d2aa592880deac5 is 50, key is test_row_0/B:col10/1731827650965/Put/seqid=0 2024-11-17T07:14:12,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741982_1158 (size=12001) 2024-11-17T07:14:12,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/f320de309ab547cc9d2aa592880deac5 2024-11-17T07:14:12,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/d8c916e744674852970fbb493c654efa is 50, key is test_row_0/C:col10/1731827650965/Put/seqid=0 2024-11-17T07:14:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-17T07:14:12,567 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:12,568 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-17T07:14:12,570 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:12,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741983_1159 (size=12001) 2024-11-17T07:14:12,571 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:12,571 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T07:14:12,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/d8c916e744674852970fbb493c654efa 2024-11-17T07:14:12,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/892bc61d5f6e40958f56f19d4f956c0a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/892bc61d5f6e40958f56f19d4f956c0a 2024-11-17T07:14:12,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/892bc61d5f6e40958f56f19d4f956c0a, entries=200, sequenceid=98, filesize=38.6 K 2024-11-17T07:14:12,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/f320de309ab547cc9d2aa592880deac5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/f320de309ab547cc9d2aa592880deac5 2024-11-17T07:14:12,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/f320de309ab547cc9d2aa592880deac5, entries=150, sequenceid=98, filesize=11.7 K 2024-11-17T07:14:12,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/d8c916e744674852970fbb493c654efa as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d8c916e744674852970fbb493c654efa 2024-11-17T07:14:12,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d8c916e744674852970fbb493c654efa, entries=150, sequenceid=98, filesize=11.7 K 2024-11-17T07:14:12,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for efcef908f9da2d14e10a65888eb3eb6d in 498ms, sequenceid=98, compaction requested=true 2024-11-17T07:14:12,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:12,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:12,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:12,607 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:12,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:12,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:12,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:12,607 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:12,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:12,608 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:12,608 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/A is initiating minor compaction (all files) 2024-11-17T07:14:12,608 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/A in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:12,608 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/a77efde900d541aabad20238bf4a3f09, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/fb37c4659212467085a079f4ccbba955, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/892bc61d5f6e40958f56f19d4f956c0a] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=99.2 K 2024-11-17T07:14:12,608 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:12,608 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/a77efde900d541aabad20238bf4a3f09, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/fb37c4659212467085a079f4ccbba955, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/892bc61d5f6e40958f56f19d4f956c0a] 2024-11-17T07:14:12,609 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:12,609 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/B is initiating minor compaction (all files) 2024-11-17T07:14:12,609 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting a77efde900d541aabad20238bf4a3f09, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731827649552 2024-11-17T07:14:12,609 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/B in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:12,609 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/ff8fa00b9eb04bfba6c6302309b49194, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/83f5157c9272425cba4ce373694db4b6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/f320de309ab547cc9d2aa592880deac5] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=35.3 K 2024-11-17T07:14:12,610 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb37c4659212467085a079f4ccbba955, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731827650287 2024-11-17T07:14:12,610 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting ff8fa00b9eb04bfba6c6302309b49194, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731827649552 2024-11-17T07:14:12,610 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 892bc61d5f6e40958f56f19d4f956c0a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731827650937 2024-11-17T07:14:12,611 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 83f5157c9272425cba4ce373694db4b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731827650287 2024-11-17T07:14:12,613 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting f320de309ab547cc9d2aa592880deac5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731827650958 2024-11-17T07:14:12,631 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#B#compaction#138 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:12,632 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/29cdea3e06f94297b492fdc1c8acf32c is 50, key is test_row_0/B:col10/1731827650965/Put/seqid=0 2024-11-17T07:14:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:12,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T07:14:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:12,632 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:12,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:12,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:12,633 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:12,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741984_1160 (size=12207) 2024-11-17T07:14:12,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827712644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,653 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111739e898260c0046008767c4a32ec7a458_efcef908f9da2d14e10a65888eb3eb6d store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,656 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111739e898260c0046008767c4a32ec7a458_efcef908f9da2d14e10a65888eb3eb6d, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:12,657 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111739e898260c0046008767c4a32ec7a458_efcef908f9da2d14e10a65888eb3eb6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:12,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827712650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827712651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827712652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179a8e765332914f1fa3df4d74c6dc64af_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827652163/Put/seqid=0 2024-11-17T07:14:12,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741986_1162 (size=14594) 2024-11-17T07:14:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T07:14:12,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741985_1161 (size=4469) 2024-11-17T07:14:12,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T07:14:12,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:12,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:12,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:12,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:12,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:12,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827712757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827712767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827712767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827712768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,770 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827712807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T07:14:12,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:12,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:12,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:12,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:12,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:12,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:12,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T07:14:12,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827712977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827712977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827712977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:12,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827712977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:12,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:12,986 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:13,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T07:14:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,052 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/29cdea3e06f94297b492fdc1c8acf32c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/29cdea3e06f94297b492fdc1c8acf32c 2024-11-17T07:14:13,057 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/B of efcef908f9da2d14e10a65888eb3eb6d into 29cdea3e06f94297b492fdc1c8acf32c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:13,057 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:13,057 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/B, priority=13, startTime=1731827652607; duration=0sec 2024-11-17T07:14:13,057 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:13,057 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:B 2024-11-17T07:14:13,058 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:13,058 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:13,058 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/C is initiating minor compaction (all files) 2024-11-17T07:14:13,058 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/C in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,059 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d3d73b550c7e4911b730e958f4a29e18, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d0dfd12e8897460e88719ae7be2058b9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d8c916e744674852970fbb493c654efa] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=35.3 K 2024-11-17T07:14:13,059 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting d3d73b550c7e4911b730e958f4a29e18, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731827649552 2024-11-17T07:14:13,059 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting d0dfd12e8897460e88719ae7be2058b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731827650287 2024-11-17T07:14:13,060 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting d8c916e744674852970fbb493c654efa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731827650958 2024-11-17T07:14:13,068 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#C#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:13,069 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/e69f4182bde44f238d6025fa4477aab6 is 50, key is test_row_0/C:col10/1731827650965/Put/seqid=0 2024-11-17T07:14:13,072 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:13,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741987_1163 (size=12207) 2024-11-17T07:14:13,080 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179a8e765332914f1fa3df4d74c6dc64af_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179a8e765332914f1fa3df4d74c6dc64af_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:13,082 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/b6cd02828b1f4248a80e72839ed0d7b5, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:13,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/b6cd02828b1f4248a80e72839ed0d7b5 is 175, key is test_row_0/A:col10/1731827652163/Put/seqid=0 2024-11-17T07:14:13,084 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#A#compaction#139 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:13,085 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/f00830eede0e4b4ab63cdc146325f346 is 175, key is test_row_0/A:col10/1731827650965/Put/seqid=0 2024-11-17T07:14:13,086 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/e69f4182bde44f238d6025fa4477aab6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/e69f4182bde44f238d6025fa4477aab6 2024-11-17T07:14:13,095 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/C of efcef908f9da2d14e10a65888eb3eb6d into e69f4182bde44f238d6025fa4477aab6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:13,095 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:13,095 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/C, priority=13, startTime=1731827652607; duration=0sec 2024-11-17T07:14:13,095 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:13,095 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:C 2024-11-17T07:14:13,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741989_1165 (size=31161) 2024-11-17T07:14:13,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741988_1164 (size=39549) 2024-11-17T07:14:13,106 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/b6cd02828b1f4248a80e72839ed0d7b5 2024-11-17T07:14:13,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/a4742ee4836e4865bf26f37d150f13c8 is 50, key is test_row_0/B:col10/1731827652163/Put/seqid=0 2024-11-17T07:14:13,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741990_1166 (size=12001) 2024-11-17T07:14:13,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T07:14:13,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:13,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,186 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T07:14:13,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827713287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:13,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827713287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:13,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827713297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:13,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:13,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827713297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:13,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:13,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,300 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:13,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827713318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:13,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T07:14:13,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:13,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T07:14:13,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:13,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,509 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/f00830eede0e4b4ab63cdc146325f346 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f00830eede0e4b4ab63cdc146325f346 2024-11-17T07:14:13,515 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/A of efcef908f9da2d14e10a65888eb3eb6d into f00830eede0e4b4ab63cdc146325f346(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:13,515 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:13,515 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/A, priority=13, startTime=1731827652606; duration=0sec 2024-11-17T07:14:13,515 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:13,515 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:A 2024-11-17T07:14:13,526 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/a4742ee4836e4865bf26f37d150f13c8 2024-11-17T07:14:13,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/5edaf5b6b0564f4e8abbc76345f6bbc9 is 50, key is test_row_0/C:col10/1731827652163/Put/seqid=0 2024-11-17T07:14:13,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741991_1167 (size=12001) 2024-11-17T07:14:13,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T07:14:13,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:13,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T07:14:13,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827713797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:13,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:13,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827713797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,800 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:13,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T07:14:13,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:13,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:13,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827713807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:13,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:13,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827713807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:13,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:13,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/5edaf5b6b0564f4e8abbc76345f6bbc9 2024-11-17T07:14:13,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/b6cd02828b1f4248a80e72839ed0d7b5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/b6cd02828b1f4248a80e72839ed0d7b5 2024-11-17T07:14:13,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T07:14:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:13,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/b6cd02828b1f4248a80e72839ed0d7b5, entries=200, sequenceid=118, filesize=38.6 K 2024-11-17T07:14:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:13,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/a4742ee4836e4865bf26f37d150f13c8 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/a4742ee4836e4865bf26f37d150f13c8 2024-11-17T07:14:13,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/a4742ee4836e4865bf26f37d150f13c8, entries=150, sequenceid=118, filesize=11.7 K 2024-11-17T07:14:13,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/5edaf5b6b0564f4e8abbc76345f6bbc9 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/5edaf5b6b0564f4e8abbc76345f6bbc9 2024-11-17T07:14:13,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/5edaf5b6b0564f4e8abbc76345f6bbc9, entries=150, sequenceid=118, filesize=11.7 K 2024-11-17T07:14:13,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for efcef908f9da2d14e10a65888eb3eb6d in 1351ms, sequenceid=118, compaction requested=false 2024-11-17T07:14:13,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:14,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-17T07:14:14,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:14,115 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T07:14:14,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:14,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:14,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:14,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:14,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:14,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:14,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117226a868d79c844fd8fa43d7312d9145a_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827652641/Put/seqid=0 2024-11-17T07:14:14,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741992_1168 (size=12204) 2024-11-17T07:14:14,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:14,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:14,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:14,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827714421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:14,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:14,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:14,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827714538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:14,543 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117226a868d79c844fd8fa43d7312d9145a_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117226a868d79c844fd8fa43d7312d9145a_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:14,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/f224ea34dc2943eaae6da0ca22e6b154, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:14,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/f224ea34dc2943eaae6da0ca22e6b154 is 175, key is test_row_0/A:col10/1731827652641/Put/seqid=0 2024-11-17T07:14:14,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741993_1169 (size=31005) 2024-11-17T07:14:14,559 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/f224ea34dc2943eaae6da0ca22e6b154 2024-11-17T07:14:14,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/2dcfa9a83daa4fbaa29d3db56d9faa52 is 50, key is test_row_0/B:col10/1731827652641/Put/seqid=0 2024-11-17T07:14:14,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741994_1170 (size=12051) 2024-11-17T07:14:14,602 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/2dcfa9a83daa4fbaa29d3db56d9faa52 2024-11-17T07:14:14,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/1c96725d310d41e38ecdc9fe615d877a is 50, key is test_row_0/C:col10/1731827652641/Put/seqid=0 2024-11-17T07:14:14,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741995_1171 (size=12051) 2024-11-17T07:14:14,648 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/1c96725d310d41e38ecdc9fe615d877a 2024-11-17T07:14:14,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/f224ea34dc2943eaae6da0ca22e6b154 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f224ea34dc2943eaae6da0ca22e6b154 2024-11-17T07:14:14,663 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f224ea34dc2943eaae6da0ca22e6b154, entries=150, sequenceid=133, filesize=30.3 K 2024-11-17T07:14:14,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/2dcfa9a83daa4fbaa29d3db56d9faa52 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/2dcfa9a83daa4fbaa29d3db56d9faa52 2024-11-17T07:14:14,673 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/2dcfa9a83daa4fbaa29d3db56d9faa52, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T07:14:14,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/1c96725d310d41e38ecdc9fe615d877a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/1c96725d310d41e38ecdc9fe615d877a 2024-11-17T07:14:14,684 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/1c96725d310d41e38ecdc9fe615d877a, entries=150, sequenceid=133, filesize=11.8 K 2024-11-17T07:14:14,686 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for efcef908f9da2d14e10a65888eb3eb6d in 572ms, sequenceid=133, compaction requested=true 2024-11-17T07:14:14,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:14,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:14,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-17T07:14:14,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-17T07:14:14,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-17T07:14:14,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1200 sec 2024-11-17T07:14:14,695 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.1250 sec 2024-11-17T07:14:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-17T07:14:14,717 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:14,718 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-17T07:14:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T07:14:14,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:14,720 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:14,721 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:14,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-17T07:14:14,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:14,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:14,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:14,775 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170396c3ba483d473e8e59eb0dd960ca90_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827654748/Put/seqid=0 2024-11-17T07:14:14,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741996_1172 (size=14794) 2024-11-17T07:14:14,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:14,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827714807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:14,813 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in efcef908f9da2d14e10a65888eb3eb6d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,813 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:14,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827714811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:14,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827714812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:14,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T07:14:14,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827714827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:14,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827714828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:14,833 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in efcef908f9da2d14e10a65888eb3eb6d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:14,834 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in efcef908f9da2d14e10a65888eb3eb6d of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,835 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:14,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T07:14:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:14,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:14,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827714928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:14,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:14,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827714928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:14,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:14,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:15,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T07:14:15,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:15,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:15,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:15,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:15,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T07:14:15,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827715137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:15,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827715138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:15,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T07:14:15,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:15,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:15,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:15,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:15,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:15,186 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:15,192 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170396c3ba483d473e8e59eb0dd960ca90_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170396c3ba483d473e8e59eb0dd960ca90_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:15,193 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/1596a6bd5df54099ba76d29d7a49f4ef, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:15,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/1596a6bd5df54099ba76d29d7a49f4ef is 175, key is test_row_0/A:col10/1731827654748/Put/seqid=0 2024-11-17T07:14:15,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741997_1173 (size=39749) 2024-11-17T07:14:15,209 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/1596a6bd5df54099ba76d29d7a49f4ef 2024-11-17T07:14:15,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/ba3361666fb54233892f1a5614b0aa54 is 50, key is test_row_0/B:col10/1731827654748/Put/seqid=0 2024-11-17T07:14:15,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741998_1174 (size=12151) 2024-11-17T07:14:15,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/ba3361666fb54233892f1a5614b0aa54 2024-11-17T07:14:15,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/52c45476eeed42e8b1da95a1f32a8b52 is 50, key is test_row_0/C:col10/1731827654748/Put/seqid=0 2024-11-17T07:14:15,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741999_1175 (size=12151) 2024-11-17T07:14:15,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/52c45476eeed42e8b1da95a1f32a8b52 2024-11-17T07:14:15,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/1596a6bd5df54099ba76d29d7a49f4ef as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/1596a6bd5df54099ba76d29d7a49f4ef 2024-11-17T07:14:15,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/1596a6bd5df54099ba76d29d7a49f4ef, entries=200, sequenceid=154, filesize=38.8 K 2024-11-17T07:14:15,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/ba3361666fb54233892f1a5614b0aa54 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/ba3361666fb54233892f1a5614b0aa54 2024-11-17T07:14:15,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/ba3361666fb54233892f1a5614b0aa54, entries=150, sequenceid=154, filesize=11.9 K 2024-11-17T07:14:15,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/52c45476eeed42e8b1da95a1f32a8b52 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/52c45476eeed42e8b1da95a1f32a8b52 2024-11-17T07:14:15,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/52c45476eeed42e8b1da95a1f32a8b52, entries=150, sequenceid=154, filesize=11.9 K 2024-11-17T07:14:15,334 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for efcef908f9da2d14e10a65888eb3eb6d in 584ms, sequenceid=154, compaction requested=true 2024-11-17T07:14:15,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:15,334 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:15,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:15,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:15,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:15,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:15,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:15,335 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:15,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:15,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-17T07:14:15,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:15,337 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:14:15,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:15,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:15,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:15,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:15,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:15,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:15,338 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:15,338 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/A is initiating minor compaction (all files) 2024-11-17T07:14:15,338 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/A in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:15,338 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f00830eede0e4b4ab63cdc146325f346, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/b6cd02828b1f4248a80e72839ed0d7b5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f224ea34dc2943eaae6da0ca22e6b154, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/1596a6bd5df54099ba76d29d7a49f4ef] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=138.1 K 2024-11-17T07:14:15,338 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:15,338 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f00830eede0e4b4ab63cdc146325f346, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/b6cd02828b1f4248a80e72839ed0d7b5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f224ea34dc2943eaae6da0ca22e6b154, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/1596a6bd5df54099ba76d29d7a49f4ef] 2024-11-17T07:14:15,340 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:15,341 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/B is initiating minor compaction (all files) 2024-11-17T07:14:15,341 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/B in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:15,341 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/29cdea3e06f94297b492fdc1c8acf32c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/a4742ee4836e4865bf26f37d150f13c8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/2dcfa9a83daa4fbaa29d3db56d9faa52, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/ba3361666fb54233892f1a5614b0aa54] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=47.3 K 2024-11-17T07:14:15,341 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting f00830eede0e4b4ab63cdc146325f346, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731827650958 2024-11-17T07:14:15,341 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 29cdea3e06f94297b492fdc1c8acf32c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731827650958 2024-11-17T07:14:15,343 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a4742ee4836e4865bf26f37d150f13c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731827652163 2024-11-17T07:14:15,343 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6cd02828b1f4248a80e72839ed0d7b5, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731827652133 2024-11-17T07:14:15,343 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dcfa9a83daa4fbaa29d3db56d9faa52, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731827652641 2024-11-17T07:14:15,343 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting f224ea34dc2943eaae6da0ca22e6b154, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731827652641 2024-11-17T07:14:15,344 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting ba3361666fb54233892f1a5614b0aa54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731827654407 2024-11-17T07:14:15,344 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1596a6bd5df54099ba76d29d7a49f4ef, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731827654402 2024-11-17T07:14:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T07:14:15,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117846e95b59efb439a90050cbad96902d7_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827654798/Put/seqid=0 2024-11-17T07:14:15,380 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#B#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:15,381 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/4ba7f39a9ae648c9b61be1f774b52a57 is 50, key is test_row_0/B:col10/1731827654748/Put/seqid=0 2024-11-17T07:14:15,389 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:15,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742000_1176 (size=12304) 2024-11-17T07:14:15,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:15,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742001_1177 (size=12493) 2024-11-17T07:14:15,420 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117846e95b59efb439a90050cbad96902d7_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117846e95b59efb439a90050cbad96902d7_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:15,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/ea5709315ff44202bc5c88ada78a4cd0, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:15,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/ea5709315ff44202bc5c88ada78a4cd0 is 175, key is test_row_0/A:col10/1731827654798/Put/seqid=0 2024-11-17T07:14:15,426 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/4ba7f39a9ae648c9b61be1f774b52a57 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/4ba7f39a9ae648c9b61be1f774b52a57 2024-11-17T07:14:15,428 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411177703fe7f4318467e9f88bd4a91e645b1_efcef908f9da2d14e10a65888eb3eb6d store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:15,431 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411177703fe7f4318467e9f88bd4a91e645b1_efcef908f9da2d14e10a65888eb3eb6d, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:15,432 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177703fe7f4318467e9f88bd4a91e645b1_efcef908f9da2d14e10a65888eb3eb6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:15,438 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/B of efcef908f9da2d14e10a65888eb3eb6d into 4ba7f39a9ae648c9b61be1f774b52a57(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:15,438 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:15,438 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/B, priority=12, startTime=1731827655335; duration=0sec 2024-11-17T07:14:15,439 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:15,439 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:B 2024-11-17T07:14:15,439 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:15,442 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:15,442 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/C is initiating minor compaction (all files) 2024-11-17T07:14:15,442 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/C in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:15,442 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/e69f4182bde44f238d6025fa4477aab6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/5edaf5b6b0564f4e8abbc76345f6bbc9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/1c96725d310d41e38ecdc9fe615d877a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/52c45476eeed42e8b1da95a1f32a8b52] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=47.3 K 2024-11-17T07:14:15,443 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting e69f4182bde44f238d6025fa4477aab6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731827650958 2024-11-17T07:14:15,443 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 5edaf5b6b0564f4e8abbc76345f6bbc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731827652163 2024-11-17T07:14:15,443 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c96725d310d41e38ecdc9fe615d877a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731827652641 2024-11-17T07:14:15,444 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 52c45476eeed42e8b1da95a1f32a8b52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731827654407 2024-11-17T07:14:15,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:15,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742002_1178 (size=31105) 2024-11-17T07:14:15,468 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=165, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/ea5709315ff44202bc5c88ada78a4cd0 2024-11-17T07:14:15,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742003_1179 (size=4469) 2024-11-17T07:14:15,492 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#C#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:15,493 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/a338e848771d4089abb5401a7b0d4e90 is 50, key is test_row_0/C:col10/1731827654748/Put/seqid=0 2024-11-17T07:14:15,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/80a4a6b817d04beaa4ce6bffa8faff6b is 50, key is test_row_0/B:col10/1731827654798/Put/seqid=0 2024-11-17T07:14:15,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742004_1180 (size=12151) 2024-11-17T07:14:15,503 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/80a4a6b817d04beaa4ce6bffa8faff6b 2024-11-17T07:14:15,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/f12a3f39477242dca103241c1e779108 is 50, key is test_row_0/C:col10/1731827654798/Put/seqid=0 2024-11-17T07:14:15,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827655497, max timestamp=1731827655510Num uniques 859; 2024-11-17T07:14:15,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:15,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827655497, max timestamp=1731827655510Num uniques 859; 2024-11-17T07:14:15,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:15,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827655497, max timestamp=1731827655510Num uniques 859; 2024-11-17T07:14:15,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:15,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742005_1181 (size=12493) 2024-11-17T07:14:15,535 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/a338e848771d4089abb5401a7b0d4e90 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/a338e848771d4089abb5401a7b0d4e90 2024-11-17T07:14:15,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742006_1182 (size=12151) 2024-11-17T07:14:15,538 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/f12a3f39477242dca103241c1e779108 2024-11-17T07:14:15,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/ea5709315ff44202bc5c88ada78a4cd0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/ea5709315ff44202bc5c88ada78a4cd0 2024-11-17T07:14:15,561 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/ea5709315ff44202bc5c88ada78a4cd0, entries=150, sequenceid=165, filesize=30.4 K 2024-11-17T07:14:15,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/80a4a6b817d04beaa4ce6bffa8faff6b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/80a4a6b817d04beaa4ce6bffa8faff6b 2024-11-17T07:14:15,575 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/80a4a6b817d04beaa4ce6bffa8faff6b, entries=150, sequenceid=165, filesize=11.9 K 2024-11-17T07:14:15,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/f12a3f39477242dca103241c1e779108 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/f12a3f39477242dca103241c1e779108 2024-11-17T07:14:15,581 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/f12a3f39477242dca103241c1e779108, entries=150, sequenceid=165, filesize=11.9 K 2024-11-17T07:14:15,582 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=120.76 KB/123660 for efcef908f9da2d14e10a65888eb3eb6d in 245ms, sequenceid=165, compaction requested=false 2024-11-17T07:14:15,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:15,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:15,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-17T07:14:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-17T07:14:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:15,584 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/C of efcef908f9da2d14e10a65888eb3eb6d into a338e848771d4089abb5401a7b0d4e90(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:15,585 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:15,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=120.76 KB heapSize=345.14 KB 2024-11-17T07:14:15,585 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/C, priority=12, startTime=1731827655335; duration=0sec 2024-11-17T07:14:15,585 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:15,585 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:C 2024-11-17T07:14:15,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:15,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:15,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:15,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-17T07:14:15,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:15,585 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 862 msec 2024-11-17T07:14:15,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:15,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:15,588 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 868 msec 2024-11-17T07:14:15,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177cee5f4177964578945543a51e5b7b1a_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827655535/Put/seqid=0 2024-11-17T07:14:15,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827715626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:15,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827715632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:15,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:15,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742007_1183 (size=17284) 2024-11-17T07:14:15,672 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:15,677 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411177cee5f4177964578945543a51e5b7b1a_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177cee5f4177964578945543a51e5b7b1a_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:15,678 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/37741dbb6c6b48d79b0a929d4d52bb3b, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:15,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/37741dbb6c6b48d79b0a929d4d52bb3b is 175, key is test_row_0/A:col10/1731827655535/Put/seqid=0 2024-11-17T07:14:15,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742008_1184 (size=48389) 2024-11-17T07:14:15,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827715737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:15,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827715747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:15,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-17T07:14:15,858 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:15,858 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:15,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-17T07:14:15,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T07:14:15,861 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:15,861 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:15,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:15,893 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#A#compaction#152 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:15,894 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/c76990b0b9404a02bd4f28705be6f02e is 175, key is test_row_0/A:col10/1731827654748/Put/seqid=0 2024-11-17T07:14:15,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742009_1185 (size=31447) 2024-11-17T07:14:15,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:15,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827715947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:15,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:15,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827715958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:15,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:15,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T07:14:16,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T07:14:16,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:16,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,102 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/37741dbb6c6b48d79b0a929d4d52bb3b 2024-11-17T07:14:16,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/6faf0b6b807e4b24b6271bee599c208a is 50, key is test_row_0/B:col10/1731827655535/Put/seqid=0 2024-11-17T07:14:16,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T07:14:16,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:16,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742010_1186 (size=12151) 2024-11-17T07:14:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T07:14:16,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:16,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827716267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:16,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827716267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:16,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T07:14:16,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:16,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,330 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/c76990b0b9404a02bd4f28705be6f02e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/c76990b0b9404a02bd4f28705be6f02e 2024-11-17T07:14:16,337 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/A of efcef908f9da2d14e10a65888eb3eb6d into c76990b0b9404a02bd4f28705be6f02e(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:16,337 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:16,337 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/A, priority=12, startTime=1731827655334; duration=0sec 2024-11-17T07:14:16,338 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:16,338 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:A 2024-11-17T07:14:16,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T07:14:16,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:16,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T07:14:16,573 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/6faf0b6b807e4b24b6271bee599c208a 2024-11-17T07:14:16,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/fb1e0bb0907840de95de8d28cfd35b4e is 50, key is test_row_0/C:col10/1731827655535/Put/seqid=0 2024-11-17T07:14:16,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T07:14:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742011_1187 (size=12151) 2024-11-17T07:14:16,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/fb1e0bb0907840de95de8d28cfd35b4e 2024-11-17T07:14:16,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/37741dbb6c6b48d79b0a929d4d52bb3b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/37741dbb6c6b48d79b0a929d4d52bb3b 2024-11-17T07:14:16,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/37741dbb6c6b48d79b0a929d4d52bb3b, entries=250, sequenceid=202, filesize=47.3 K 2024-11-17T07:14:16,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/6faf0b6b807e4b24b6271bee599c208a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/6faf0b6b807e4b24b6271bee599c208a 2024-11-17T07:14:16,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/6faf0b6b807e4b24b6271bee599c208a, entries=150, sequenceid=202, filesize=11.9 K 2024-11-17T07:14:16,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/fb1e0bb0907840de95de8d28cfd35b4e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/fb1e0bb0907840de95de8d28cfd35b4e 2024-11-17T07:14:16,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/fb1e0bb0907840de95de8d28cfd35b4e, entries=150, sequenceid=202, filesize=11.9 K 2024-11-17T07:14:16,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~345.09 KB/353376, currentSize=60.38 KB/61830 for efcef908f9da2d14e10a65888eb3eb6d in 1099ms, sequenceid=202, compaction requested=true 2024-11-17T07:14:16,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:16,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:16,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:16,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:16,684 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:16,684 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:16,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:16,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:16,685 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:16,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:16,685 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/A is initiating minor compaction (all files) 2024-11-17T07:14:16,686 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/A in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,686 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/c76990b0b9404a02bd4f28705be6f02e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/ea5709315ff44202bc5c88ada78a4cd0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/37741dbb6c6b48d79b0a929d4d52bb3b] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=108.3 K 2024-11-17T07:14:16,686 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,686 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/c76990b0b9404a02bd4f28705be6f02e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/ea5709315ff44202bc5c88ada78a4cd0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/37741dbb6c6b48d79b0a929d4d52bb3b] 2024-11-17T07:14:16,686 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:16,686 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/B is initiating minor compaction (all files) 2024-11-17T07:14:16,686 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/B in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,686 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/4ba7f39a9ae648c9b61be1f774b52a57, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/80a4a6b817d04beaa4ce6bffa8faff6b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/6faf0b6b807e4b24b6271bee599c208a] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=35.9 K 2024-11-17T07:14:16,688 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting c76990b0b9404a02bd4f28705be6f02e, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731827654407 2024-11-17T07:14:16,689 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ba7f39a9ae648c9b61be1f774b52a57, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731827654407 2024-11-17T07:14:16,692 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea5709315ff44202bc5c88ada78a4cd0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731827654798 2024-11-17T07:14:16,693 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 80a4a6b817d04beaa4ce6bffa8faff6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731827654798 2024-11-17T07:14:16,693 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37741dbb6c6b48d79b0a929d4d52bb3b, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731827655535 2024-11-17T07:14:16,693 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 6faf0b6b807e4b24b6271bee599c208a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731827655535 2024-11-17T07:14:16,715 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#B#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:16,716 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/dbb480108a934ee8b57971fb3e11ab93 is 50, key is test_row_0/B:col10/1731827655535/Put/seqid=0 2024-11-17T07:14:16,721 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:16,726 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111797c6aa0e8b764137abbd5c6767540a14_efcef908f9da2d14e10a65888eb3eb6d store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:16,729 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111797c6aa0e8b764137abbd5c6767540a14_efcef908f9da2d14e10a65888eb3eb6d, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:16,729 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111797c6aa0e8b764137abbd5c6767540a14_efcef908f9da2d14e10a65888eb3eb6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:16,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742012_1188 (size=12595) 2024-11-17T07:14:16,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742013_1189 (size=4469) 2024-11-17T07:14:16,753 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#A#compaction#160 average throughput is 0.76 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:16,753 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/5cc3ee6681d3476bb25662d5d4617674 is 175, key is test_row_0/A:col10/1731827655535/Put/seqid=0 2024-11-17T07:14:16,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742014_1190 (size=31549) 2024-11-17T07:14:16,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:16,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T07:14:16,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:16,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:16,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:16,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:16,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:16,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:16,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117db2cf67578f6484d8e44b4fdb335c6ea_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827655622/Put/seqid=0 2024-11-17T07:14:16,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T07:14:16,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742015_1191 (size=14794) 2024-11-17T07:14:16,807 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:16,811 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117db2cf67578f6484d8e44b4fdb335c6ea_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117db2cf67578f6484d8e44b4fdb335c6ea_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:16,812 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/00e5b147817349f4a4f5420ca2377fa6, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:16,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/00e5b147817349f4a4f5420ca2377fa6 is 175, key is test_row_0/A:col10/1731827655622/Put/seqid=0 2024-11-17T07:14:16,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742016_1192 (size=39749) 2024-11-17T07:14:16,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827716836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:16,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:16,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827716837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:16,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:16,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827716840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,843 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:16,844 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in efcef908f9da2d14e10a65888eb3eb6d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,845 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:16,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827716847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:16,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:16,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827716847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:16,853 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in efcef908f9da2d14e10a65888eb3eb6d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:16,854 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in efcef908f9da2d14e10a65888eb3eb6d of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:16,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827716947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:16,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:16,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827716947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:16,950 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:16,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:16,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:16,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T07:14:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:16,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:16,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T07:14:17,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T07:14:17,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:17,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:17,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:17,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:17,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:17,159 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/dbb480108a934ee8b57971fb3e11ab93 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/dbb480108a934ee8b57971fb3e11ab93 2024-11-17T07:14:17,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:17,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827717157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:17,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:17,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:17,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827717158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:17,168 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/B of efcef908f9da2d14e10a65888eb3eb6d into dbb480108a934ee8b57971fb3e11ab93(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:17,168 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:17,168 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/B, priority=13, startTime=1731827656684; duration=0sec 2024-11-17T07:14:17,169 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:17,169 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:B 2024-11-17T07:14:17,170 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:17,174 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:17,174 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/C is initiating minor compaction (all files) 2024-11-17T07:14:17,174 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/C in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:17,174 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/a338e848771d4089abb5401a7b0d4e90, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/f12a3f39477242dca103241c1e779108, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/fb1e0bb0907840de95de8d28cfd35b4e] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=35.9 K 2024-11-17T07:14:17,175 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a338e848771d4089abb5401a7b0d4e90, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731827654407 2024-11-17T07:14:17,175 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/5cc3ee6681d3476bb25662d5d4617674 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5cc3ee6681d3476bb25662d5d4617674 2024-11-17T07:14:17,176 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting f12a3f39477242dca103241c1e779108, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731827654798 2024-11-17T07:14:17,176 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting fb1e0bb0907840de95de8d28cfd35b4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731827655535 2024-11-17T07:14:17,183 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/A of efcef908f9da2d14e10a65888eb3eb6d into 5cc3ee6681d3476bb25662d5d4617674(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:17,183 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:17,183 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/A, priority=13, startTime=1731827656684; duration=0sec 2024-11-17T07:14:17,183 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:17,183 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:A 2024-11-17T07:14:17,188 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#C#compaction#162 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:17,188 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/cc9d18c416c84a47bfc596cd25e8f4b2 is 50, key is test_row_0/C:col10/1731827655535/Put/seqid=0 2024-11-17T07:14:17,218 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/00e5b147817349f4a4f5420ca2377fa6 2024-11-17T07:14:17,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742017_1193 (size=12595) 2024-11-17T07:14:17,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/6a6898f02f584b39ab1b5ae840710730 is 50, key is test_row_0/B:col10/1731827655622/Put/seqid=0 2024-11-17T07:14:17,237 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/cc9d18c416c84a47bfc596cd25e8f4b2 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/cc9d18c416c84a47bfc596cd25e8f4b2 2024-11-17T07:14:17,245 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/C of efcef908f9da2d14e10a65888eb3eb6d into cc9d18c416c84a47bfc596cd25e8f4b2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:17,245 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:17,245 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/C, priority=13, startTime=1731827656685; duration=0sec 2024-11-17T07:14:17,245 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:17,245 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:C 2024-11-17T07:14:17,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742018_1194 (size=12151) 2024-11-17T07:14:17,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/6a6898f02f584b39ab1b5ae840710730 2024-11-17T07:14:17,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T07:14:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:17,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:17,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:17,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/68a48e8e55424ebc9b6361a5edcb4828 is 50, key is test_row_0/C:col10/1731827655622/Put/seqid=0 2024-11-17T07:14:17,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742019_1195 (size=12151) 2024-11-17T07:14:17,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T07:14:17,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:17,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:17,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:17,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:17,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:17,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827717477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:17,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827717477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:17,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T07:14:17,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:17,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:17,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:17,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:17,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:17,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:17,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/68a48e8e55424ebc9b6361a5edcb4828 2024-11-17T07:14:17,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/00e5b147817349f4a4f5420ca2377fa6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/00e5b147817349f4a4f5420ca2377fa6 2024-11-17T07:14:17,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/00e5b147817349f4a4f5420ca2377fa6, entries=200, sequenceid=216, filesize=38.8 K 2024-11-17T07:14:17,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/6a6898f02f584b39ab1b5ae840710730 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/6a6898f02f584b39ab1b5ae840710730 2024-11-17T07:14:17,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/6a6898f02f584b39ab1b5ae840710730, entries=150, sequenceid=216, filesize=11.9 K 2024-11-17T07:14:17,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/68a48e8e55424ebc9b6361a5edcb4828 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/68a48e8e55424ebc9b6361a5edcb4828 2024-11-17T07:14:17,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/68a48e8e55424ebc9b6361a5edcb4828, entries=150, sequenceid=216, filesize=11.9 K 2024-11-17T07:14:17,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for efcef908f9da2d14e10a65888eb3eb6d in 936ms, sequenceid=216, compaction requested=false 2024-11-17T07:14:17,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:17,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-17T07:14:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:17,724 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T07:14:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:17,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:17,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f77f030eae1b4b6f840a45734ae45ffa_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827656834/Put/seqid=0 2024-11-17T07:14:17,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742020_1196 (size=12304) 2024-11-17T07:14:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:17,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T07:14:18,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827718008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:18,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827718009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:18,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827718127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:18,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827718128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:18,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,190 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f77f030eae1b4b6f840a45734ae45ffa_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f77f030eae1b4b6f840a45734ae45ffa_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:18,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/5fe4e5cfeb434cf0a49d0650ac896137, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:18,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/5fe4e5cfeb434cf0a49d0650ac896137 is 175, key is test_row_0/A:col10/1731827656834/Put/seqid=0 2024-11-17T07:14:18,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742021_1197 (size=31105) 2024-11-17T07:14:18,237 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/5fe4e5cfeb434cf0a49d0650ac896137 2024-11-17T07:14:18,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/533036d0df3a4ca39d57cd42ccec663d is 50, key is test_row_0/B:col10/1731827656834/Put/seqid=0 2024-11-17T07:14:18,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742022_1198 (size=12151) 2024-11-17T07:14:18,267 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/533036d0df3a4ca39d57cd42ccec663d 2024-11-17T07:14:18,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/697af042c7914dda994ebe1ceaf5ea25 is 50, key is test_row_0/C:col10/1731827656834/Put/seqid=0 2024-11-17T07:14:18,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742023_1199 (size=12151) 2024-11-17T07:14:18,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827718337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:18,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:18,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827718337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:18,479 INFO [master/1dd62e24b303:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-17T07:14:18,479 INFO [master/1dd62e24b303:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-17T07:14:18,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:18,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827718647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:18,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:18,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827718648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:18,713 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/697af042c7914dda994ebe1ceaf5ea25 2024-11-17T07:14:18,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/5fe4e5cfeb434cf0a49d0650ac896137 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5fe4e5cfeb434cf0a49d0650ac896137 2024-11-17T07:14:18,723 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5fe4e5cfeb434cf0a49d0650ac896137, entries=150, sequenceid=238, filesize=30.4 K 2024-11-17T07:14:18,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/533036d0df3a4ca39d57cd42ccec663d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/533036d0df3a4ca39d57cd42ccec663d 2024-11-17T07:14:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,730 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/533036d0df3a4ca39d57cd42ccec663d, entries=150, sequenceid=238, filesize=11.9 K 2024-11-17T07:14:18,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/697af042c7914dda994ebe1ceaf5ea25 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/697af042c7914dda994ebe1ceaf5ea25 2024-11-17T07:14:18,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,736 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/697af042c7914dda994ebe1ceaf5ea25, entries=150, sequenceid=238, filesize=11.9 K 2024-11-17T07:14:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,737 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for efcef908f9da2d14e10a65888eb3eb6d in 1013ms, sequenceid=238, compaction requested=true 2024-11-17T07:14:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:18,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:18,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-17T07:14:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-17T07:14:18,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-17T07:14:18,741 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8770 sec 2024-11-17T07:14:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.8840 sec 2024-11-17T07:14:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:18,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:19,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T07:14:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:19,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:19,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:19,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:19,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:19,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d2a499c48eba45a386e79121dc735687_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827659160/Put/seqid=0 2024-11-17T07:14:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742024_1200 (size=12304) 2024-11-17T07:14:19,221 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:19,227 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d2a499c48eba45a386e79121dc735687_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d2a499c48eba45a386e79121dc735687_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:19,228 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/eaa0ede739d6406fb64f8dfcc0f53348, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:19,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/eaa0ede739d6406fb64f8dfcc0f53348 is 175, key is test_row_0/A:col10/1731827659160/Put/seqid=0 2024-11-17T07:14:19,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:19,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827719245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:19,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:19,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827719246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:19,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742025_1201 (size=31105) 2024-11-17T07:14:19,263 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/eaa0ede739d6406fb64f8dfcc0f53348 2024-11-17T07:14:19,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/9ad7c702bb0441d987e10664295403d8 is 50, key is test_row_0/B:col10/1731827659160/Put/seqid=0 2024-11-17T07:14:19,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742026_1202 (size=12151) 2024-11-17T07:14:19,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/9ad7c702bb0441d987e10664295403d8 2024-11-17T07:14:19,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/777785a4ba9d4d579c95ba1bd1c503a2 is 50, key is test_row_0/C:col10/1731827659160/Put/seqid=0 2024-11-17T07:14:19,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827719357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:19,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827719357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:19,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742027_1203 (size=12151) 2024-11-17T07:14:19,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:19,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827719567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:19,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827719568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:19,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/777785a4ba9d4d579c95ba1bd1c503a2 2024-11-17T07:14:19,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/eaa0ede739d6406fb64f8dfcc0f53348 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/eaa0ede739d6406fb64f8dfcc0f53348 2024-11-17T07:14:19,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/eaa0ede739d6406fb64f8dfcc0f53348, entries=150, sequenceid=253, filesize=30.4 K 2024-11-17T07:14:19,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/9ad7c702bb0441d987e10664295403d8 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/9ad7c702bb0441d987e10664295403d8 2024-11-17T07:14:19,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/9ad7c702bb0441d987e10664295403d8, entries=150, sequenceid=253, filesize=11.9 K 2024-11-17T07:14:19,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/777785a4ba9d4d579c95ba1bd1c503a2 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/777785a4ba9d4d579c95ba1bd1c503a2 2024-11-17T07:14:19,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/777785a4ba9d4d579c95ba1bd1c503a2, entries=150, sequenceid=253, filesize=11.9 K 2024-11-17T07:14:19,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for efcef908f9da2d14e10a65888eb3eb6d in 669ms, sequenceid=253, compaction requested=true 2024-11-17T07:14:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:19,831 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:19,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:19,831 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:19,835 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133508 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:19,835 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/A is initiating minor compaction (all files) 2024-11-17T07:14:19,836 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/A in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:19,836 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5cc3ee6681d3476bb25662d5d4617674, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/00e5b147817349f4a4f5420ca2377fa6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5fe4e5cfeb434cf0a49d0650ac896137, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/eaa0ede739d6406fb64f8dfcc0f53348] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=130.4 K 2024-11-17T07:14:19,836 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:19,836 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5cc3ee6681d3476bb25662d5d4617674, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/00e5b147817349f4a4f5420ca2377fa6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5fe4e5cfeb434cf0a49d0650ac896137, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/eaa0ede739d6406fb64f8dfcc0f53348] 2024-11-17T07:14:19,836 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:19,836 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/B is initiating minor compaction (all files) 2024-11-17T07:14:19,836 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/B in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:19,836 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/dbb480108a934ee8b57971fb3e11ab93, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/6a6898f02f584b39ab1b5ae840710730, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/533036d0df3a4ca39d57cd42ccec663d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/9ad7c702bb0441d987e10664295403d8] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=47.9 K 2024-11-17T07:14:19,836 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cc3ee6681d3476bb25662d5d4617674, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731827655535 2024-11-17T07:14:19,837 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting dbb480108a934ee8b57971fb3e11ab93, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731827655535 2024-11-17T07:14:19,837 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00e5b147817349f4a4f5420ca2377fa6, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731827655610 2024-11-17T07:14:19,837 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a6898f02f584b39ab1b5ae840710730, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731827655622 2024-11-17T07:14:19,837 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fe4e5cfeb434cf0a49d0650ac896137, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731827656822 2024-11-17T07:14:19,838 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 533036d0df3a4ca39d57cd42ccec663d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731827656822 2024-11-17T07:14:19,838 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaa0ede739d6406fb64f8dfcc0f53348, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731827658006 2024-11-17T07:14:19,838 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ad7c702bb0441d987e10664295403d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731827658006 2024-11-17T07:14:19,859 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:19,876 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#B#compaction#172 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:19,877 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/f1fc88b9d2444f72aa5cb30334f205e1 is 50, key is test_row_0/B:col10/1731827659160/Put/seqid=0 2024-11-17T07:14:19,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T07:14:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:19,897 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117edc3722d763f4223bbb71c23d4ed9d92_efcef908f9da2d14e10a65888eb3eb6d store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:19,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:19,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:19,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:19,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:19,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:19,897 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:19,900 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117edc3722d763f4223bbb71c23d4ed9d92_efcef908f9da2d14e10a65888eb3eb6d, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:19,900 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117edc3722d763f4223bbb71c23d4ed9d92_efcef908f9da2d14e10a65888eb3eb6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:19,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:19,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827719951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:19,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827719954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:19,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:19,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742028_1204 (size=12731) 2024-11-17T07:14:19,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117aec8f495240745d5a6abd9e4fddb3bb2_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827659888/Put/seqid=0 2024-11-17T07:14:19,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742029_1205 (size=4469) 2024-11-17T07:14:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-17T07:14:20,027 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:20,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:20,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-17T07:14:20,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T07:14:20,030 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:20,031 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:20,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:20,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742030_1206 (size=14994) 2024-11-17T07:14:20,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827720058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:20,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827720077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:20,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T07:14:20,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T07:14:20,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:20,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:20,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:20,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:20,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827720277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:20,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:20,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827720287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:20,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T07:14:20,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:20,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:20,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:20,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T07:14:20,377 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/f1fc88b9d2444f72aa5cb30334f205e1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/f1fc88b9d2444f72aa5cb30334f205e1 2024-11-17T07:14:20,384 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/B of efcef908f9da2d14e10a65888eb3eb6d into f1fc88b9d2444f72aa5cb30334f205e1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:20,384 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:20,384 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/B, priority=12, startTime=1731827659831; duration=0sec 2024-11-17T07:14:20,384 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:20,384 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:B 2024-11-17T07:14:20,384 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:20,386 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:20,386 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/C is initiating minor compaction (all files) 2024-11-17T07:14:20,386 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/C in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:20,386 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/cc9d18c416c84a47bfc596cd25e8f4b2, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/68a48e8e55424ebc9b6361a5edcb4828, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/697af042c7914dda994ebe1ceaf5ea25, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/777785a4ba9d4d579c95ba1bd1c503a2] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=47.9 K 2024-11-17T07:14:20,387 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting cc9d18c416c84a47bfc596cd25e8f4b2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731827655535 2024-11-17T07:14:20,387 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 68a48e8e55424ebc9b6361a5edcb4828, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731827655622 2024-11-17T07:14:20,388 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 697af042c7914dda994ebe1ceaf5ea25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731827656822 2024-11-17T07:14:20,390 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 777785a4ba9d4d579c95ba1bd1c503a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731827658006 2024-11-17T07:14:20,393 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#A#compaction#171 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:20,394 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/3440faf204694765afe6f4b01ce061fe is 175, key is test_row_0/A:col10/1731827659160/Put/seqid=0 2024-11-17T07:14:20,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742031_1207 (size=31685) 2024-11-17T07:14:20,415 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#C#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:20,416 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/b1aaa98de1fe494dbeb4f8d138f9e67f is 50, key is test_row_0/C:col10/1731827659160/Put/seqid=0 2024-11-17T07:14:20,423 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/3440faf204694765afe6f4b01ce061fe as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/3440faf204694765afe6f4b01ce061fe 2024-11-17T07:14:20,432 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/A of efcef908f9da2d14e10a65888eb3eb6d into 3440faf204694765afe6f4b01ce061fe(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:20,432 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:20,432 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/A, priority=12, startTime=1731827659831; duration=0sec 2024-11-17T07:14:20,440 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:20,440 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:A 2024-11-17T07:14:20,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742032_1208 (size=12731) 2024-11-17T07:14:20,452 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:20,462 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117aec8f495240745d5a6abd9e4fddb3bb2_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117aec8f495240745d5a6abd9e4fddb3bb2_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:20,464 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/15ab773879de4f01a9d1f495be6429e1, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:20,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/15ab773879de4f01a9d1f495be6429e1 is 175, key is test_row_0/A:col10/1731827659888/Put/seqid=0 2024-11-17T07:14:20,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742033_1209 (size=39949) 2024-11-17T07:14:20,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T07:14:20,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:20,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:20,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:20,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827720597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827720598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:20,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T07:14:20,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:20,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:20,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:20,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T07:14:20,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T07:14:20,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:20,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:20,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:20,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,856 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/b1aaa98de1fe494dbeb4f8d138f9e67f as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/b1aaa98de1fe494dbeb4f8d138f9e67f 2024-11-17T07:14:20,862 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/C of efcef908f9da2d14e10a65888eb3eb6d into b1aaa98de1fe494dbeb4f8d138f9e67f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:20,862 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:20,862 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/C, priority=12, startTime=1731827659831; duration=0sec 2024-11-17T07:14:20,862 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:20,862 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:C 2024-11-17T07:14:20,878 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/15ab773879de4f01a9d1f495be6429e1 2024-11-17T07:14:20,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:20,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53080 deadline: 1731827720877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:20,884 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in efcef908f9da2d14e10a65888eb3eb6d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:20,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/97176c6783a0441ba412adb169603940 is 50, key is test_row_0/B:col10/1731827659888/Put/seqid=0 2024-11-17T07:14:20,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742034_1210 (size=12301) 2024-11-17T07:14:20,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53076 deadline: 1731827720887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:20,891 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in efcef908f9da2d14e10a65888eb3eb6d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:20,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:20,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53120 deadline: 1731827720897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:20,901 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in efcef908f9da2d14e10a65888eb3eb6d of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:20,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T07:14:20,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:20,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:20,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:20,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:20,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:21,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T07:14:21,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:21,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:21,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:21,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:21,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:21,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:21,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827721117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:21,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827721117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:21,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T07:14:21,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T07:14:21,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:21,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:21,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:21,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:21,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:21,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/97176c6783a0441ba412adb169603940 2024-11-17T07:14:21,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/2f9f7b9f2b41435b984a01b097e2b171 is 50, key is test_row_0/C:col10/1731827659888/Put/seqid=0 2024-11-17T07:14:21,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742035_1211 (size=12301) 2024-11-17T07:14:21,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/2f9f7b9f2b41435b984a01b097e2b171 2024-11-17T07:14:21,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/15ab773879de4f01a9d1f495be6429e1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/15ab773879de4f01a9d1f495be6429e1 2024-11-17T07:14:21,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/15ab773879de4f01a9d1f495be6429e1, entries=200, sequenceid=272, filesize=39.0 K 2024-11-17T07:14:21,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/97176c6783a0441ba412adb169603940 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/97176c6783a0441ba412adb169603940 2024-11-17T07:14:21,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/97176c6783a0441ba412adb169603940, entries=150, sequenceid=272, filesize=12.0 K 2024-11-17T07:14:21,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/2f9f7b9f2b41435b984a01b097e2b171 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/2f9f7b9f2b41435b984a01b097e2b171 2024-11-17T07:14:21,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/2f9f7b9f2b41435b984a01b097e2b171, entries=150, sequenceid=272, filesize=12.0 K 2024-11-17T07:14:21,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for efcef908f9da2d14e10a65888eb3eb6d in 1456ms, sequenceid=272, compaction requested=false 2024-11-17T07:14:21,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:21,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-17T07:14:21,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:21,413 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T07:14:21,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:21,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:21,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:21,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:21,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:21,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:21,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111780da8e5a621240e2b6b229581b95b746_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827659925/Put/seqid=0 2024-11-17T07:14:21,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742036_1212 (size=12454) 2024-11-17T07:14:21,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,435 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111780da8e5a621240e2b6b229581b95b746_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111780da8e5a621240e2b6b229581b95b746_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:21,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/f219f3320d734ccaa23e304c3349ea90, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:21,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/f219f3320d734ccaa23e304c3349ea90 is 175, key is test_row_0/A:col10/1731827659925/Put/seqid=0 2024-11-17T07:14:21,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742037_1213 (size=31255) 2024-11-17T07:14:21,857 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/f219f3320d734ccaa23e304c3349ea90 2024-11-17T07:14:21,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/98271dc6c59e43dc9ffb16a24dd152ab is 50, key is test_row_0/B:col10/1731827659925/Put/seqid=0 2024-11-17T07:14:21,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742038_1214 (size=12301) 2024-11-17T07:14:21,887 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/98271dc6c59e43dc9ffb16a24dd152ab 2024-11-17T07:14:21,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/4b2bb09e92fd44e7ae11c3cc7a075c32 is 50, key is test_row_0/C:col10/1731827659925/Put/seqid=0 2024-11-17T07:14:21,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742039_1215 (size=12301) 2024-11-17T07:14:21,914 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/4b2bb09e92fd44e7ae11c3cc7a075c32 2024-11-17T07:14:21,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/f219f3320d734ccaa23e304c3349ea90 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f219f3320d734ccaa23e304c3349ea90 2024-11-17T07:14:21,931 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f219f3320d734ccaa23e304c3349ea90, entries=150, sequenceid=288, filesize=30.5 K 2024-11-17T07:14:21,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/98271dc6c59e43dc9ffb16a24dd152ab as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/98271dc6c59e43dc9ffb16a24dd152ab 2024-11-17T07:14:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,938 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/98271dc6c59e43dc9ffb16a24dd152ab, entries=150, sequenceid=288, filesize=12.0 K 2024-11-17T07:14:21,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/4b2bb09e92fd44e7ae11c3cc7a075c32 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/4b2bb09e92fd44e7ae11c3cc7a075c32 2024-11-17T07:14:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,947 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/4b2bb09e92fd44e7ae11c3cc7a075c32, entries=150, sequenceid=288, filesize=12.0 K 2024-11-17T07:14:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,948 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for efcef908f9da2d14e10a65888eb3eb6d in 535ms, sequenceid=288, compaction requested=true 2024-11-17T07:14:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:21,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-17T07:14:21,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-17T07:14:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-17T07:14:21,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9190 sec 2024-11-17T07:14:21,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.9260 sec 2024-11-17T07:14:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:21,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:22,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:14:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:22,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:22,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:22,167 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:22,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:22,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-17T07:14:22,178 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:22,178 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-17T07:14:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T07:14:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,183 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:22,184 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:22,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ccf31bdb7def405facf61381430b9805_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827662155/Put/seqid=0 2024-11-17T07:14:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742040_1216 (size=22618) 2024-11-17T07:14:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731827662168, max timestamp=1731827662263Num uniques 859; 2024-11-17T07:14:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-17T07:14:22,265 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731827662168, max timestamp=1731827662263Num uniques 859; 2024-11-17T07:14:22,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-17T07:14:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731827662168, max timestamp=1731827662263Num uniques 859; 2024-11-17T07:14:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-17T07:14:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T07:14:22,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827722287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:22,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827722288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:22,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T07:14:22,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:22,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:22,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:22,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827722397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827722397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:22,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T07:14:22,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:22,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:22,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:22,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T07:14:22,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827722608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:22,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827722609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:22,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T07:14:22,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:22,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:22,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:22,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,652 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:22,657 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ccf31bdb7def405facf61381430b9805_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ccf31bdb7def405facf61381430b9805_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:22,658 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/7f42f2a3bc8b4a29bef60fd55154c54f, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:22,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/7f42f2a3bc8b4a29bef60fd55154c54f is 175, key is test_row_0/A:col10/1731827662155/Put/seqid=0 2024-11-17T07:14:22,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742041_1217 (size=66023) 2024-11-17T07:14:22,691 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/7f42f2a3bc8b4a29bef60fd55154c54f 2024-11-17T07:14:22,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/70b48cb1397344c6984bc97498880dcc is 50, key is test_row_0/B:col10/1731827662155/Put/seqid=0 2024-11-17T07:14:22,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742042_1218 (size=12301) 2024-11-17T07:14:22,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/70b48cb1397344c6984bc97498880dcc 2024-11-17T07:14:22,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/13bdbe4e8ebb47b4ae5a1bd0a7112dcc is 50, key is test_row_0/C:col10/1731827662155/Put/seqid=0 2024-11-17T07:14:22,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742043_1219 (size=12301) 2024-11-17T07:14:22,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T07:14:22,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:22,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:22,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:22,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T07:14:22,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827722917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827722917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:22,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:22,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T07:14:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:22,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:23,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T07:14:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:23,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:23,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:23,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:23,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:23,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/13bdbe4e8ebb47b4ae5a1bd0a7112dcc 2024-11-17T07:14:23,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/7f42f2a3bc8b4a29bef60fd55154c54f as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7f42f2a3bc8b4a29bef60fd55154c54f 2024-11-17T07:14:23,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7f42f2a3bc8b4a29bef60fd55154c54f, entries=350, sequenceid=298, filesize=64.5 K 2024-11-17T07:14:23,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/70b48cb1397344c6984bc97498880dcc as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/70b48cb1397344c6984bc97498880dcc 2024-11-17T07:14:23,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/70b48cb1397344c6984bc97498880dcc, entries=150, sequenceid=298, filesize=12.0 K 2024-11-17T07:14:23,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/13bdbe4e8ebb47b4ae5a1bd0a7112dcc as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/13bdbe4e8ebb47b4ae5a1bd0a7112dcc 2024-11-17T07:14:23,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/13bdbe4e8ebb47b4ae5a1bd0a7112dcc, entries=150, sequenceid=298, filesize=12.0 K 2024-11-17T07:14:23,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for efcef908f9da2d14e10a65888eb3eb6d in 1020ms, sequenceid=298, compaction requested=true 2024-11-17T07:14:23,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:23,188 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:23,189 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 168912 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:23,189 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/A is initiating minor compaction (all files) 2024-11-17T07:14:23,189 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/A in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:23,190 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/3440faf204694765afe6f4b01ce061fe, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/15ab773879de4f01a9d1f495be6429e1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f219f3320d734ccaa23e304c3349ea90, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7f42f2a3bc8b4a29bef60fd55154c54f] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=165.0 K 2024-11-17T07:14:23,190 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:23,190 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/3440faf204694765afe6f4b01ce061fe, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/15ab773879de4f01a9d1f495be6429e1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f219f3320d734ccaa23e304c3349ea90, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7f42f2a3bc8b4a29bef60fd55154c54f] 2024-11-17T07:14:23,190 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3440faf204694765afe6f4b01ce061fe, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731827658006 2024-11-17T07:14:23,191 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15ab773879de4f01a9d1f495be6429e1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731827659227 2024-11-17T07:14:23,191 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting f219f3320d734ccaa23e304c3349ea90, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731827659925 2024-11-17T07:14:23,191 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f42f2a3bc8b4a29bef60fd55154c54f, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731827662134 2024-11-17T07:14:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:23,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:23,199 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:23,200 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:23,202 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:23,202 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/B is initiating minor compaction (all files) 2024-11-17T07:14:23,202 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/B in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:23,202 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/f1fc88b9d2444f72aa5cb30334f205e1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/97176c6783a0441ba412adb169603940, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/98271dc6c59e43dc9ffb16a24dd152ab, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/70b48cb1397344c6984bc97498880dcc] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=48.5 K 2024-11-17T07:14:23,203 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting f1fc88b9d2444f72aa5cb30334f205e1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731827658006 2024-11-17T07:14:23,203 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 97176c6783a0441ba412adb169603940, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731827659227 2024-11-17T07:14:23,204 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 98271dc6c59e43dc9ffb16a24dd152ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731827659925 2024-11-17T07:14:23,204 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 70b48cb1397344c6984bc97498880dcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731827662136 2024-11-17T07:14:23,205 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111780852098e0a84fb6bed77a28719a7a3f_efcef908f9da2d14e10a65888eb3eb6d store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:23,208 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111780852098e0a84fb6bed77a28719a7a3f_efcef908f9da2d14e10a65888eb3eb6d, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:23,208 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111780852098e0a84fb6bed77a28719a7a3f_efcef908f9da2d14e10a65888eb3eb6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:23,228 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#B#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:23,229 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/544048d34fbb488ca9938a63621b7443 is 50, key is test_row_0/B:col10/1731827662155/Put/seqid=0 2024-11-17T07:14:23,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742044_1220 (size=4469) 2024-11-17T07:14:23,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742045_1221 (size=13017) 2024-11-17T07:14:23,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-17T07:14:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:23,260 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-11-17T07:14:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:23,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:23,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c17f040639384eb5b05146167a9f7ffc_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827662277/Put/seqid=0 2024-11-17T07:14:23,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742046_1222 (size=12454) 2024-11-17T07:14:23,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:23,304 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c17f040639384eb5b05146167a9f7ffc_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c17f040639384eb5b05146167a9f7ffc_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:23,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/300fe268e05140da855e5ad4cccfbf84, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:23,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/300fe268e05140da855e5ad4cccfbf84 is 175, key is test_row_0/A:col10/1731827662277/Put/seqid=0 2024-11-17T07:14:23,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T07:14:23,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742047_1223 (size=31255) 2024-11-17T07:14:23,322 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/300fe268e05140da855e5ad4cccfbf84 2024-11-17T07:14:23,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/c77a147f372344578b60e9e94ed38a52 is 50, key is test_row_0/B:col10/1731827662277/Put/seqid=0 2024-11-17T07:14:23,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742048_1224 (size=12301) 2024-11-17T07:14:23,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:23,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:23,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:23,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827723444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:23,448 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:23,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827723448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:23,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:23,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:23,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827723557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:23,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:23,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:23,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:23,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827723567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:23,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:23,641 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#A#compaction#183 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:23,641 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/592ce602a65142c589b1426bc2990c8d is 175, key is test_row_0/A:col10/1731827662155/Put/seqid=0 2024-11-17T07:14:23,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742049_1225 (size=31971) 2024-11-17T07:14:23,662 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/544048d34fbb488ca9938a63621b7443 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/544048d34fbb488ca9938a63621b7443 2024-11-17T07:14:23,666 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/592ce602a65142c589b1426bc2990c8d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/592ce602a65142c589b1426bc2990c8d 2024-11-17T07:14:23,673 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/B of efcef908f9da2d14e10a65888eb3eb6d into 544048d34fbb488ca9938a63621b7443(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:23,673 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:23,673 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/B, priority=12, startTime=1731827663199; duration=0sec 2024-11-17T07:14:23,673 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:23,673 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:B 2024-11-17T07:14:23,673 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:23,676 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/A of efcef908f9da2d14e10a65888eb3eb6d into 592ce602a65142c589b1426bc2990c8d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:23,677 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:23,677 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/A, priority=12, startTime=1731827663188; duration=0sec 2024-11-17T07:14:23,677 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:23,677 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:A 2024-11-17T07:14:23,679 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:23,680 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/C is initiating minor compaction (all files) 2024-11-17T07:14:23,680 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/C in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:23,680 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/b1aaa98de1fe494dbeb4f8d138f9e67f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/2f9f7b9f2b41435b984a01b097e2b171, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/4b2bb09e92fd44e7ae11c3cc7a075c32, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/13bdbe4e8ebb47b4ae5a1bd0a7112dcc] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=48.5 K 2024-11-17T07:14:23,681 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting b1aaa98de1fe494dbeb4f8d138f9e67f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731827658006 2024-11-17T07:14:23,681 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f9f7b9f2b41435b984a01b097e2b171, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731827659227 2024-11-17T07:14:23,683 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b2bb09e92fd44e7ae11c3cc7a075c32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731827659925 2024-11-17T07:14:23,683 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 13bdbe4e8ebb47b4ae5a1bd0a7112dcc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731827662136 2024-11-17T07:14:23,693 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#C#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:23,694 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/630531e43a84422ebb28b522896fe7dd is 50, key is test_row_0/C:col10/1731827662155/Put/seqid=0 2024-11-17T07:14:23,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742050_1226 (size=13017) 2024-11-17T07:14:23,719 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/630531e43a84422ebb28b522896fe7dd as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/630531e43a84422ebb28b522896fe7dd 2024-11-17T07:14:23,730 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/C of efcef908f9da2d14e10a65888eb3eb6d into 630531e43a84422ebb28b522896fe7dd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:23,730 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:23,730 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/C, priority=12, startTime=1731827663199; duration=0sec 2024-11-17T07:14:23,731 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:23,731 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:C 2024-11-17T07:14:23,750 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/c77a147f372344578b60e9e94ed38a52 2024-11-17T07:14:23,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/ca292d2060f24d56b80cccfc48b1104e is 50, key is test_row_0/C:col10/1731827662277/Put/seqid=0 2024-11-17T07:14:23,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827723767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:23,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:23,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827723779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:23,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:23,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742051_1227 (size=12301) 2024-11-17T07:14:23,795 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/ca292d2060f24d56b80cccfc48b1104e 2024-11-17T07:14:23,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/300fe268e05140da855e5ad4cccfbf84 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/300fe268e05140da855e5ad4cccfbf84 2024-11-17T07:14:23,810 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/300fe268e05140da855e5ad4cccfbf84, entries=150, sequenceid=324, filesize=30.5 K 2024-11-17T07:14:23,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/c77a147f372344578b60e9e94ed38a52 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/c77a147f372344578b60e9e94ed38a52 2024-11-17T07:14:23,819 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/c77a147f372344578b60e9e94ed38a52, entries=150, sequenceid=324, filesize=12.0 K 2024-11-17T07:14:23,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/ca292d2060f24d56b80cccfc48b1104e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/ca292d2060f24d56b80cccfc48b1104e 2024-11-17T07:14:23,839 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/ca292d2060f24d56b80cccfc48b1104e, entries=150, sequenceid=324, filesize=12.0 K 2024-11-17T07:14:23,840 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=46.96 KB/48090 for efcef908f9da2d14e10a65888eb3eb6d in 580ms, sequenceid=324, compaction requested=false 2024-11-17T07:14:23,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:23,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:23,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-17T07:14:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-17T07:14:23,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-17T07:14:23,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6580 sec 2024-11-17T07:14:23,847 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.6660 sec 2024-11-17T07:14:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:24,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:14:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:24,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:24,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117100a28ed256a4c7cacfb1a54d1e8f05c_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827663442/Put/seqid=0 2024-11-17T07:14:24,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742052_1228 (size=14994) 2024-11-17T07:14:24,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:24,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827724133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:24,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827664097, max timestamp=1731827664136Num uniques 859; 2024-11-17T07:14:24,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:14:24,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827664097, max timestamp=1731827664136Num uniques 859; 2024-11-17T07:14:24,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:14:24,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827664097, max timestamp=1731827664136Num uniques 859; 2024-11-17T07:14:24,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:14:24,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:24,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827724207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:24,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,213 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:24,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:24,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827724247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,253 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-17T07:14:24,332 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:24,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827724327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:24,337 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-17T07:14:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T07:14:24,342 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:24,343 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:24,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T07:14:24,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:24,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827724457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:24,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:24,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T07:14:24,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:24,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:24,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:24,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:24,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:24,505 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:24,513 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117100a28ed256a4c7cacfb1a54d1e8f05c_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117100a28ed256a4c7cacfb1a54d1e8f05c_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:24,514 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/7697c63f5e5243bc855b5bfd8fcf735c, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:24,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/7697c63f5e5243bc855b5bfd8fcf735c is 175, key is test_row_0/A:col10/1731827663442/Put/seqid=0 2024-11-17T07:14:24,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742053_1229 (size=39949) 2024-11-17T07:14:24,520 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/7697c63f5e5243bc855b5bfd8fcf735c 2024-11-17T07:14:24,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/32403413fc8349a2bbc3298445882ce8 is 50, key is test_row_0/B:col10/1731827663442/Put/seqid=0 2024-11-17T07:14:24,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742054_1230 (size=12301) 2024-11-17T07:14:24,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:24,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827724547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:24,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T07:14:24,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:24,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:24,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:24,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:24,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T07:14:24,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827724767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:24,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T07:14:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:24,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:24,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:24,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827724857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:24,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:24,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/32403413fc8349a2bbc3298445882ce8 2024-11-17T07:14:24,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/8a97a04551954d1b942241aa9750f22a is 50, key is test_row_0/C:col10/1731827663442/Put/seqid=0 2024-11-17T07:14:24,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T07:14:24,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:24,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:24,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:24,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:24,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:24,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742055_1231 (size=12301) 2024-11-17T07:14:24,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/8a97a04551954d1b942241aa9750f22a 2024-11-17T07:14:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T07:14:24,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/7697c63f5e5243bc855b5bfd8fcf735c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7697c63f5e5243bc855b5bfd8fcf735c 2024-11-17T07:14:24,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7697c63f5e5243bc855b5bfd8fcf735c, entries=200, sequenceid=338, filesize=39.0 K 2024-11-17T07:14:24,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/32403413fc8349a2bbc3298445882ce8 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/32403413fc8349a2bbc3298445882ce8 2024-11-17T07:14:24,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/32403413fc8349a2bbc3298445882ce8, entries=150, sequenceid=338, filesize=12.0 K 2024-11-17T07:14:24,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/8a97a04551954d1b942241aa9750f22a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/8a97a04551954d1b942241aa9750f22a 2024-11-17T07:14:24,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/8a97a04551954d1b942241aa9750f22a, entries=150, sequenceid=338, filesize=12.0 K 2024-11-17T07:14:24,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for efcef908f9da2d14e10a65888eb3eb6d in 906ms, sequenceid=338, compaction requested=true 2024-11-17T07:14:24,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:24,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:24,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:24,987 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:24,987 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:24,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:24,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:24,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:24,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:24,989 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:24,989 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:24,989 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/B is initiating minor compaction (all files) 2024-11-17T07:14:24,989 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/A is initiating minor compaction (all files) 2024-11-17T07:14:24,989 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/B in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:24,989 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/A in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:24,989 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/544048d34fbb488ca9938a63621b7443, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/c77a147f372344578b60e9e94ed38a52, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/32403413fc8349a2bbc3298445882ce8] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=36.7 K 2024-11-17T07:14:24,989 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/592ce602a65142c589b1426bc2990c8d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/300fe268e05140da855e5ad4cccfbf84, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7697c63f5e5243bc855b5bfd8fcf735c] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=100.8 K 2024-11-17T07:14:24,989 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:24,989 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/592ce602a65142c589b1426bc2990c8d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/300fe268e05140da855e5ad4cccfbf84, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7697c63f5e5243bc855b5bfd8fcf735c] 2024-11-17T07:14:24,989 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 544048d34fbb488ca9938a63621b7443, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731827662136 2024-11-17T07:14:24,990 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 592ce602a65142c589b1426bc2990c8d, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731827662136 2024-11-17T07:14:24,990 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting c77a147f372344578b60e9e94ed38a52, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731827662277 2024-11-17T07:14:24,990 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 300fe268e05140da855e5ad4cccfbf84, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731827662277 2024-11-17T07:14:24,990 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 32403413fc8349a2bbc3298445882ce8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731827663442 2024-11-17T07:14:24,990 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7697c63f5e5243bc855b5bfd8fcf735c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731827663430 2024-11-17T07:14:24,999 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#B#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:24,999 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/703ee3e078b44fe3aa615a4ccfa1492b is 50, key is test_row_0/B:col10/1731827663442/Put/seqid=0 2024-11-17T07:14:24,999 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:25,008 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411175faacdc27dd3478a8b0905f5c53ff2e3_efcef908f9da2d14e10a65888eb3eb6d store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:25,010 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411175faacdc27dd3478a8b0905f5c53ff2e3_efcef908f9da2d14e10a65888eb3eb6d, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:25,010 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411175faacdc27dd3478a8b0905f5c53ff2e3_efcef908f9da2d14e10a65888eb3eb6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:25,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742056_1232 (size=13119) 2024-11-17T07:14:25,017 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/703ee3e078b44fe3aa615a4ccfa1492b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/703ee3e078b44fe3aa615a4ccfa1492b 2024-11-17T07:14:25,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742057_1233 (size=4469) 2024-11-17T07:14:25,023 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/B of efcef908f9da2d14e10a65888eb3eb6d into 703ee3e078b44fe3aa615a4ccfa1492b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:25,023 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:25,023 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/B, priority=13, startTime=1731827664987; duration=0sec 2024-11-17T07:14:25,023 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:25,023 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:B 2024-11-17T07:14:25,023 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:25,024 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:25,025 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/C is initiating minor compaction (all files) 2024-11-17T07:14:25,025 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/C in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:25,025 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/630531e43a84422ebb28b522896fe7dd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/ca292d2060f24d56b80cccfc48b1104e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/8a97a04551954d1b942241aa9750f22a] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=36.7 K 2024-11-17T07:14:25,025 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 630531e43a84422ebb28b522896fe7dd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731827662136 2024-11-17T07:14:25,026 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting ca292d2060f24d56b80cccfc48b1104e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731827662277 2024-11-17T07:14:25,026 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a97a04551954d1b942241aa9750f22a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731827663442 2024-11-17T07:14:25,032 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#C#compaction#194 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:25,033 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/8ff790bd48114358a24a6a1c703076c1 is 50, key is test_row_0/C:col10/1731827663442/Put/seqid=0 2024-11-17T07:14:25,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742058_1234 (size=13119) 2024-11-17T07:14:25,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-17T07:14:25,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:25,117 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-17T07:14:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:25,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:25,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d327cd31085848799bd02009e3dc0b1a_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827664189/Put/seqid=0 2024-11-17T07:14:25,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742059_1235 (size=12454) 2024-11-17T07:14:25,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:25,137 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d327cd31085848799bd02009e3dc0b1a_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d327cd31085848799bd02009e3dc0b1a_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:25,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/8a4fd0ebf14540b29c6918e44e83a181, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:25,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/8a4fd0ebf14540b29c6918e44e83a181 is 175, key is test_row_0/A:col10/1731827664189/Put/seqid=0 2024-11-17T07:14:25,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742060_1236 (size=31255) 2024-11-17T07:14:25,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:25,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:25,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:25,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827725310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:25,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827725377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:25,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:25,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:25,383 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:25,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827725417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:25,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:25,422 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#A#compaction#193 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:25,423 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/5a2d373fe61941d2a9d19a2e7c5c37e4 is 175, key is test_row_0/A:col10/1731827663442/Put/seqid=0 2024-11-17T07:14:25,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742061_1237 (size=32073) 2024-11-17T07:14:25,442 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/8ff790bd48114358a24a6a1c703076c1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/8ff790bd48114358a24a6a1c703076c1 2024-11-17T07:14:25,446 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/C of efcef908f9da2d14e10a65888eb3eb6d into 8ff790bd48114358a24a6a1c703076c1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:25,446 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:25,447 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/C, priority=13, startTime=1731827664988; duration=0sec 2024-11-17T07:14:25,447 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:25,447 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:C 2024-11-17T07:14:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T07:14:25,545 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=373, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/8a4fd0ebf14540b29c6918e44e83a181 2024-11-17T07:14:25,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/8b2c0a7d01484d6ba2ca9a9144c88191 is 50, key is test_row_0/B:col10/1731827664189/Put/seqid=0 2024-11-17T07:14:25,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742062_1238 (size=12301) 2024-11-17T07:14:25,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827725627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:25,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:25,833 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/5a2d373fe61941d2a9d19a2e7c5c37e4 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5a2d373fe61941d2a9d19a2e7c5c37e4 2024-11-17T07:14:25,839 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/A of efcef908f9da2d14e10a65888eb3eb6d into 5a2d373fe61941d2a9d19a2e7c5c37e4(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:25,839 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:25,839 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/A, priority=13, startTime=1731827664987; duration=0sec 2024-11-17T07:14:25,839 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:25,839 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:A 2024-11-17T07:14:25,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827725937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:25,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:25,959 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/8b2c0a7d01484d6ba2ca9a9144c88191 2024-11-17T07:14:25,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/82f966c6ce7c4fe0a3f40f84fede16dc is 50, key is test_row_0/C:col10/1731827664189/Put/seqid=0 2024-11-17T07:14:25,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742063_1239 (size=12301) 2024-11-17T07:14:26,376 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/82f966c6ce7c4fe0a3f40f84fede16dc 2024-11-17T07:14:26,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/8a4fd0ebf14540b29c6918e44e83a181 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/8a4fd0ebf14540b29c6918e44e83a181 2024-11-17T07:14:26,385 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/8a4fd0ebf14540b29c6918e44e83a181, entries=150, sequenceid=373, filesize=30.5 K 2024-11-17T07:14:26,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/8b2c0a7d01484d6ba2ca9a9144c88191 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/8b2c0a7d01484d6ba2ca9a9144c88191 2024-11-17T07:14:26,390 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/8b2c0a7d01484d6ba2ca9a9144c88191, entries=150, sequenceid=373, filesize=12.0 K 2024-11-17T07:14:26,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/82f966c6ce7c4fe0a3f40f84fede16dc as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/82f966c6ce7c4fe0a3f40f84fede16dc 2024-11-17T07:14:26,395 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/82f966c6ce7c4fe0a3f40f84fede16dc, entries=150, sequenceid=373, filesize=12.0 K 2024-11-17T07:14:26,396 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for efcef908f9da2d14e10a65888eb3eb6d in 1280ms, sequenceid=373, compaction requested=false 2024-11-17T07:14:26,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:26,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:26,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-17T07:14:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-17T07:14:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:26,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T07:14:26,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:26,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:26,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:26,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:26,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:26,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:26,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-17T07:14:26,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0550 sec 2024-11-17T07:14:26,405 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.0660 sec 2024-11-17T07:14:26,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ce2a53148b28475f968f6a063da887f7_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827665284/Put/seqid=0 2024-11-17T07:14:26,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742064_1240 (size=14994) 2024-11-17T07:14:26,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827726453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:26,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:26,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:26,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827726455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:26,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-17T07:14:26,488 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:26,488 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-17T07:14:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T07:14:26,490 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:26,491 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:26,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:26,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827726557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:26,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827726567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:26,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:26,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:26,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:26,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T07:14:26,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T07:14:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:26,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:26,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:26,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:26,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827726767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:26,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:26,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:26,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:26,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:26,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827726777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:26,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:26,796 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T07:14:26,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:26,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:26,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:26,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:26,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:26,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T07:14:26,816 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:26,819 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ce2a53148b28475f968f6a063da887f7_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ce2a53148b28475f968f6a063da887f7_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:26,820 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/2fc5659a759f438b8cc13c64c85cbaef, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:26,821 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/2fc5659a759f438b8cc13c64c85cbaef is 175, key is test_row_0/A:col10/1731827665284/Put/seqid=0 2024-11-17T07:14:26,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742065_1241 (size=39949) 2024-11-17T07:14:26,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T07:14:26,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:26,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:26,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:26,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:26,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:26,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:27,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827727077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:27,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:27,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:27,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:27,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:27,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827727087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:27,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:27,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T07:14:27,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:27,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:27,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:27,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T07:14:27,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/2fc5659a759f438b8cc13c64c85cbaef 2024-11-17T07:14:27,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/d740302d54fd4281ae289b3790c9faee is 50, key is test_row_0/B:col10/1731827665284/Put/seqid=0 2024-11-17T07:14:27,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742066_1242 (size=12301) 2024-11-17T07:14:27,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T07:14:27,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:27,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:27,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:27,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T07:14:27,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:27,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:27,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:27,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T07:14:27,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:27,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53072 deadline: 1731827727586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:27,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:27,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:27,591 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:27,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53104 deadline: 1731827727597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:27,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:14:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T07:14:27,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/d740302d54fd4281ae289b3790c9faee 2024-11-17T07:14:27,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/2926492b21b84dbd9ecca4a79b72c9aa is 50, key is test_row_0/C:col10/1731827665284/Put/seqid=0 2024-11-17T07:14:27,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742067_1243 (size=12301) 2024-11-17T07:14:27,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T07:14:27,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:27,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:27,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:27,719 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T07:14:27,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:27,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:27,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:27,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:28,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T07:14:28,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:28,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:28,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:28,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:28,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:28,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:28,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/2926492b21b84dbd9ecca4a79b72c9aa 2024-11-17T07:14:28,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/2fc5659a759f438b8cc13c64c85cbaef as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/2fc5659a759f438b8cc13c64c85cbaef 2024-11-17T07:14:28,063 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/2fc5659a759f438b8cc13c64c85cbaef, entries=200, sequenceid=387, filesize=39.0 K 2024-11-17T07:14:28,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/d740302d54fd4281ae289b3790c9faee as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/d740302d54fd4281ae289b3790c9faee 2024-11-17T07:14:28,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/d740302d54fd4281ae289b3790c9faee, entries=150, sequenceid=387, filesize=12.0 K 2024-11-17T07:14:28,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/2926492b21b84dbd9ecca4a79b72c9aa as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/2926492b21b84dbd9ecca4a79b72c9aa 2024-11-17T07:14:28,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/2926492b21b84dbd9ecca4a79b72c9aa, entries=150, sequenceid=387, filesize=12.0 K 2024-11-17T07:14:28,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for efcef908f9da2d14e10a65888eb3eb6d in 1674ms, sequenceid=387, compaction requested=true 2024-11-17T07:14:28,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:28,074 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:28,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:28,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:28,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:28,074 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:28,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:28,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store efcef908f9da2d14e10a65888eb3eb6d:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:28,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:28,075 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:28,075 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/A is initiating minor compaction (all files) 2024-11-17T07:14:28,075 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/A in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:28,075 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5a2d373fe61941d2a9d19a2e7c5c37e4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/8a4fd0ebf14540b29c6918e44e83a181, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/2fc5659a759f438b8cc13c64c85cbaef] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=100.9 K 2024-11-17T07:14:28,075 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:28,075 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5a2d373fe61941d2a9d19a2e7c5c37e4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/8a4fd0ebf14540b29c6918e44e83a181, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/2fc5659a759f438b8cc13c64c85cbaef] 2024-11-17T07:14:28,076 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:28,076 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/B is initiating minor compaction (all files) 2024-11-17T07:14:28,076 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/B in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:28,076 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/703ee3e078b44fe3aa615a4ccfa1492b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/8b2c0a7d01484d6ba2ca9a9144c88191, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/d740302d54fd4281ae289b3790c9faee] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=36.8 K 2024-11-17T07:14:28,076 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a2d373fe61941d2a9d19a2e7c5c37e4, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731827663442 2024-11-17T07:14:28,079 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a4fd0ebf14540b29c6918e44e83a181, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731827664189 2024-11-17T07:14:28,080 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fc5659a759f438b8cc13c64c85cbaef, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731827665284 2024-11-17T07:14:28,081 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 703ee3e078b44fe3aa615a4ccfa1492b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731827663442 2024-11-17T07:14:28,083 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b2c0a7d01484d6ba2ca9a9144c88191, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731827664189 2024-11-17T07:14:28,083 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting d740302d54fd4281ae289b3790c9faee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731827665284 2024-11-17T07:14:28,090 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:28,093 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111715fa962a1cea4c4189c1b2763ebb7c02_efcef908f9da2d14e10a65888eb3eb6d store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:28,093 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#B#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:28,094 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/fd7302eba0a949cfaa977e5eb3b0c88e is 50, key is test_row_0/B:col10/1731827665284/Put/seqid=0 2024-11-17T07:14:28,095 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111715fa962a1cea4c4189c1b2763ebb7c02_efcef908f9da2d14e10a65888eb3eb6d, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:28,095 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111715fa962a1cea4c4189c1b2763ebb7c02_efcef908f9da2d14e10a65888eb3eb6d because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:28,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742068_1244 (size=13221) 2024-11-17T07:14:28,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742069_1245 (size=4469) 2024-11-17T07:14:28,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-17T07:14:28,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:28,179 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T07:14:28,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:28,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:28,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:28,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:28,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:28,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:28,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117254fe249f5db4af78b27c4c3fa882f7d_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827666444/Put/seqid=0 2024-11-17T07:14:28,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742070_1246 (size=12454) 2024-11-17T07:14:28,268 INFO [Thread-682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-682. 2024-11-17T07:14:28,268 DEBUG [Thread-682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:28,269 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:28,269 DEBUG [Thread-682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:28,269 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:28,271 INFO [Thread-688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-688. 2024-11-17T07:14:28,271 DEBUG [Thread-688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:28,271 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:28,271 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:28,271 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:28,272 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-17T07:14:28,272 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:28,272 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:28,272 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:28,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:28,275 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-17T07:14:28,276 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:28,276 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:28,276 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:28,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:28,501 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#A#compaction#201 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:28,501 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/47f6c1436e7d4baab05d5cc0bfbefeba is 175, key is test_row_0/A:col10/1731827665284/Put/seqid=0 2024-11-17T07:14:28,504 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/fd7302eba0a949cfaa977e5eb3b0c88e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/fd7302eba0a949cfaa977e5eb3b0c88e 2024-11-17T07:14:28,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742071_1247 (size=32175) 2024-11-17T07:14:28,509 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/B of efcef908f9da2d14e10a65888eb3eb6d into fd7302eba0a949cfaa977e5eb3b0c88e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:28,509 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:28,509 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/B, priority=13, startTime=1731827668074; duration=0sec 2024-11-17T07:14:28,509 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:28,509 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:B 2024-11-17T07:14:28,509 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:28,510 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:28,510 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): efcef908f9da2d14e10a65888eb3eb6d/C is initiating minor compaction (all files) 2024-11-17T07:14:28,510 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of efcef908f9da2d14e10a65888eb3eb6d/C in TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:28,510 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/8ff790bd48114358a24a6a1c703076c1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/82f966c6ce7c4fe0a3f40f84fede16dc, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/2926492b21b84dbd9ecca4a79b72c9aa] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp, totalSize=36.8 K 2024-11-17T07:14:28,510 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ff790bd48114358a24a6a1c703076c1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731827663442 2024-11-17T07:14:28,511 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 82f966c6ce7c4fe0a3f40f84fede16dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731827664189 2024-11-17T07:14:28,511 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2926492b21b84dbd9ecca4a79b72c9aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731827665284 2024-11-17T07:14:28,517 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): efcef908f9da2d14e10a65888eb3eb6d#C#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:28,517 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/18f98b80ac004f8eb8491c185296aaac is 50, key is test_row_0/C:col10/1731827665284/Put/seqid=0 2024-11-17T07:14:28,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742072_1248 (size=13221) 2024-11-17T07:14:28,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:28,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. as already flushing 2024-11-17T07:14:28,599 INFO [Thread-671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-671. 2024-11-17T07:14:28,599 DEBUG [Thread-671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:28,599 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:28,599 DEBUG [Thread-671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:28,599 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:28,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:28,606 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117254fe249f5db4af78b27c4c3fa882f7d_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117254fe249f5db4af78b27c4c3fa882f7d_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:28,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/60f2904c3fa54c21beb301191ff3f815, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:28,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/60f2904c3fa54c21beb301191ff3f815 is 175, key is test_row_0/A:col10/1731827666444/Put/seqid=0 2024-11-17T07:14:28,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742073_1249 (size=31255) 2024-11-17T07:14:28,618 INFO [Thread-675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-675. 2024-11-17T07:14:28,618 DEBUG [Thread-675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:28,618 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:28,618 DEBUG [Thread-675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:28,618 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:28,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T07:14:28,909 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/47f6c1436e7d4baab05d5cc0bfbefeba as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/47f6c1436e7d4baab05d5cc0bfbefeba 2024-11-17T07:14:28,914 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/A of efcef908f9da2d14e10a65888eb3eb6d into 47f6c1436e7d4baab05d5cc0bfbefeba(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:28,914 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:28,914 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/A, priority=13, startTime=1731827668074; duration=0sec 2024-11-17T07:14:28,914 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:28,914 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:A 2024-11-17T07:14:28,925 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/18f98b80ac004f8eb8491c185296aaac as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/18f98b80ac004f8eb8491c185296aaac 2024-11-17T07:14:28,929 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in efcef908f9da2d14e10a65888eb3eb6d/C of efcef908f9da2d14e10a65888eb3eb6d into 18f98b80ac004f8eb8491c185296aaac(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:28,929 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:28,929 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., storeName=efcef908f9da2d14e10a65888eb3eb6d/C, priority=13, startTime=1731827668074; duration=0sec 2024-11-17T07:14:28,929 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:28,930 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: efcef908f9da2d14e10a65888eb3eb6d:C 2024-11-17T07:14:29,012 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=407, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/60f2904c3fa54c21beb301191ff3f815 2024-11-17T07:14:29,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/d28b0cb92404457d9a83107415d3548d is 50, key is test_row_0/B:col10/1731827666444/Put/seqid=0 2024-11-17T07:14:29,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742074_1250 (size=12301) 2024-11-17T07:14:29,424 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/d28b0cb92404457d9a83107415d3548d 2024-11-17T07:14:29,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/6feb0fc6495547b7b4e65699757b1f69 is 50, key is test_row_0/C:col10/1731827666444/Put/seqid=0 2024-11-17T07:14:29,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742075_1251 (size=12301) 2024-11-17T07:14:29,835 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/6feb0fc6495547b7b4e65699757b1f69 2024-11-17T07:14:29,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/60f2904c3fa54c21beb301191ff3f815 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/60f2904c3fa54c21beb301191ff3f815 2024-11-17T07:14:29,842 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/60f2904c3fa54c21beb301191ff3f815, entries=150, sequenceid=407, filesize=30.5 K 2024-11-17T07:14:29,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/d28b0cb92404457d9a83107415d3548d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/d28b0cb92404457d9a83107415d3548d 2024-11-17T07:14:29,846 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/d28b0cb92404457d9a83107415d3548d, entries=150, sequenceid=407, filesize=12.0 K 2024-11-17T07:14:29,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/6feb0fc6495547b7b4e65699757b1f69 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/6feb0fc6495547b7b4e65699757b1f69 2024-11-17T07:14:29,850 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/6feb0fc6495547b7b4e65699757b1f69, entries=150, sequenceid=407, filesize=12.0 K 2024-11-17T07:14:29,851 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=13.42 KB/13740 for efcef908f9da2d14e10a65888eb3eb6d in 1672ms, sequenceid=407, compaction requested=false 2024-11-17T07:14:29,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for efcef908f9da2d14e10a65888eb3eb6d: 2024-11-17T07:14:29,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:29,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-17T07:14:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-17T07:14:29,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-17T07:14:29,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3610 sec 2024-11-17T07:14:29,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 3.3660 sec 2024-11-17T07:14:30,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-17T07:14:30,647 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:30,918 INFO [Thread-677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-677. 2024-11-17T07:14:30,918 DEBUG [Thread-677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:30,918 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:30,919 DEBUG [Thread-677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:30,919 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:30,949 INFO [Thread-679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-679. 2024-11-17T07:14:30,949 DEBUG [Thread-679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:30,949 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:30,949 DEBUG [Thread-679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:30,949 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:31,008 INFO [Thread-673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-673. 2024-11-17T07:14:31,008 DEBUG [Thread-673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:31,008 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:31,008 DEBUG [Thread-673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:31,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 187 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2704 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2627 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1151 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3453 rows 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1149 2024-11-17T07:14:31,009 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3447 rows 2024-11-17T07:14:31,009 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T07:14:31,009 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T07:14:31,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:31,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:31,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:31,010 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T07:14:31,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T07:14:31,013 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827671012"}]},"ts":"1731827671012"} 2024-11-17T07:14:31,014 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T07:14:31,014 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T07:14:31,014 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T07:14:31,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, UNASSIGN}] 2024-11-17T07:14:31,017 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, UNASSIGN 2024-11-17T07:14:31,017 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=efcef908f9da2d14e10a65888eb3eb6d, regionState=CLOSING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:31,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, UNASSIGN because future has completed 2024-11-17T07:14:31,019 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T07:14:31,019 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:14:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T07:14:31,171 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:31,171 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T07:14:31,171 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing efcef908f9da2d14e10a65888eb3eb6d, disabling compactions & flushes 2024-11-17T07:14:31,171 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:31,171 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:31,171 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. after waiting 0 ms 2024-11-17T07:14:31,171 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:31,171 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing efcef908f9da2d14e10a65888eb3eb6d 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T07:14:31,171 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=A 2024-11-17T07:14:31,171 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:31,171 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=B 2024-11-17T07:14:31,171 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:31,171 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK efcef908f9da2d14e10a65888eb3eb6d, store=C 2024-11-17T07:14:31,171 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:31,177 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111753abfa2330c74cbdb0f9288731412149_efcef908f9da2d14e10a65888eb3eb6d is 50, key is test_row_0/A:col10/1731827670917/Put/seqid=0 2024-11-17T07:14:31,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742076_1252 (size=12454) 2024-11-17T07:14:31,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T07:14:31,581 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:31,585 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111753abfa2330c74cbdb0f9288731412149_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111753abfa2330c74cbdb0f9288731412149_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:31,586 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/9167257199494169a81c4178ae7bc9b0, store: [table=TestAcidGuarantees family=A region=efcef908f9da2d14e10a65888eb3eb6d] 2024-11-17T07:14:31,587 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/9167257199494169a81c4178ae7bc9b0 is 175, key is test_row_0/A:col10/1731827670917/Put/seqid=0 2024-11-17T07:14:31,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742077_1253 (size=31255) 2024-11-17T07:14:31,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T07:14:31,991 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=418, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/9167257199494169a81c4178ae7bc9b0 2024-11-17T07:14:31,998 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/02b54bbad3db435aba16794edd5881e6 is 50, key is test_row_0/B:col10/1731827670917/Put/seqid=0 2024-11-17T07:14:32,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742078_1254 (size=12301) 2024-11-17T07:14:32,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T07:14:32,403 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/02b54bbad3db435aba16794edd5881e6 2024-11-17T07:14:32,409 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/61693a32479643418692748182fe137a is 50, key is test_row_0/C:col10/1731827670917/Put/seqid=0 2024-11-17T07:14:32,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742079_1255 (size=12301) 2024-11-17T07:14:32,814 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/61693a32479643418692748182fe137a 2024-11-17T07:14:32,818 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/A/9167257199494169a81c4178ae7bc9b0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/9167257199494169a81c4178ae7bc9b0 2024-11-17T07:14:32,822 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/9167257199494169a81c4178ae7bc9b0, entries=150, sequenceid=418, filesize=30.5 K 2024-11-17T07:14:32,822 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/B/02b54bbad3db435aba16794edd5881e6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/02b54bbad3db435aba16794edd5881e6 2024-11-17T07:14:32,826 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/02b54bbad3db435aba16794edd5881e6, entries=150, sequenceid=418, filesize=12.0 K 2024-11-17T07:14:32,826 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/.tmp/C/61693a32479643418692748182fe137a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/61693a32479643418692748182fe137a 2024-11-17T07:14:32,830 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/61693a32479643418692748182fe137a, entries=150, sequenceid=418, filesize=12.0 K 2024-11-17T07:14:32,830 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for efcef908f9da2d14e10a65888eb3eb6d in 1659ms, sequenceid=418, compaction requested=true 2024-11-17T07:14:32,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/6c7302d343a04535961ed4f04694b404, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/e00872c4b3474f00b9e0e3dd87638bbf, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/14be745c2a91483c9d5886d454a181ea, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/a77efde900d541aabad20238bf4a3f09, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/fb37c4659212467085a079f4ccbba955, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/892bc61d5f6e40958f56f19d4f956c0a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f00830eede0e4b4ab63cdc146325f346, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/b6cd02828b1f4248a80e72839ed0d7b5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f224ea34dc2943eaae6da0ca22e6b154, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/1596a6bd5df54099ba76d29d7a49f4ef, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/c76990b0b9404a02bd4f28705be6f02e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/ea5709315ff44202bc5c88ada78a4cd0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/37741dbb6c6b48d79b0a929d4d52bb3b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5cc3ee6681d3476bb25662d5d4617674, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/00e5b147817349f4a4f5420ca2377fa6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5fe4e5cfeb434cf0a49d0650ac896137, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/3440faf204694765afe6f4b01ce061fe, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/eaa0ede739d6406fb64f8dfcc0f53348, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/15ab773879de4f01a9d1f495be6429e1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f219f3320d734ccaa23e304c3349ea90, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7f42f2a3bc8b4a29bef60fd55154c54f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/592ce602a65142c589b1426bc2990c8d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/300fe268e05140da855e5ad4cccfbf84, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7697c63f5e5243bc855b5bfd8fcf735c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5a2d373fe61941d2a9d19a2e7c5c37e4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/8a4fd0ebf14540b29c6918e44e83a181, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/2fc5659a759f438b8cc13c64c85cbaef] to archive 2024-11-17T07:14:32,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:14:32,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/6c7302d343a04535961ed4f04694b404 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/6c7302d343a04535961ed4f04694b404 2024-11-17T07:14:32,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/e00872c4b3474f00b9e0e3dd87638bbf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/e00872c4b3474f00b9e0e3dd87638bbf 2024-11-17T07:14:32,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/14be745c2a91483c9d5886d454a181ea to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/14be745c2a91483c9d5886d454a181ea 2024-11-17T07:14:32,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/a77efde900d541aabad20238bf4a3f09 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/a77efde900d541aabad20238bf4a3f09 2024-11-17T07:14:32,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/fb37c4659212467085a079f4ccbba955 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/fb37c4659212467085a079f4ccbba955 2024-11-17T07:14:32,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/892bc61d5f6e40958f56f19d4f956c0a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/892bc61d5f6e40958f56f19d4f956c0a 2024-11-17T07:14:32,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f00830eede0e4b4ab63cdc146325f346 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f00830eede0e4b4ab63cdc146325f346 2024-11-17T07:14:32,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/b6cd02828b1f4248a80e72839ed0d7b5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/b6cd02828b1f4248a80e72839ed0d7b5 2024-11-17T07:14:32,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f224ea34dc2943eaae6da0ca22e6b154 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f224ea34dc2943eaae6da0ca22e6b154 2024-11-17T07:14:32,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/1596a6bd5df54099ba76d29d7a49f4ef to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/1596a6bd5df54099ba76d29d7a49f4ef 2024-11-17T07:14:32,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/c76990b0b9404a02bd4f28705be6f02e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/c76990b0b9404a02bd4f28705be6f02e 2024-11-17T07:14:32,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/ea5709315ff44202bc5c88ada78a4cd0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/ea5709315ff44202bc5c88ada78a4cd0 2024-11-17T07:14:32,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/37741dbb6c6b48d79b0a929d4d52bb3b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/37741dbb6c6b48d79b0a929d4d52bb3b 2024-11-17T07:14:32,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5cc3ee6681d3476bb25662d5d4617674 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5cc3ee6681d3476bb25662d5d4617674 2024-11-17T07:14:32,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/00e5b147817349f4a4f5420ca2377fa6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/00e5b147817349f4a4f5420ca2377fa6 2024-11-17T07:14:32,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5fe4e5cfeb434cf0a49d0650ac896137 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5fe4e5cfeb434cf0a49d0650ac896137 2024-11-17T07:14:32,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/3440faf204694765afe6f4b01ce061fe to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/3440faf204694765afe6f4b01ce061fe 2024-11-17T07:14:32,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/eaa0ede739d6406fb64f8dfcc0f53348 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/eaa0ede739d6406fb64f8dfcc0f53348 2024-11-17T07:14:32,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/15ab773879de4f01a9d1f495be6429e1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/15ab773879de4f01a9d1f495be6429e1 2024-11-17T07:14:32,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f219f3320d734ccaa23e304c3349ea90 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/f219f3320d734ccaa23e304c3349ea90 2024-11-17T07:14:32,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7f42f2a3bc8b4a29bef60fd55154c54f to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7f42f2a3bc8b4a29bef60fd55154c54f 2024-11-17T07:14:32,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/592ce602a65142c589b1426bc2990c8d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/592ce602a65142c589b1426bc2990c8d 2024-11-17T07:14:32,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/300fe268e05140da855e5ad4cccfbf84 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/300fe268e05140da855e5ad4cccfbf84 2024-11-17T07:14:32,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7697c63f5e5243bc855b5bfd8fcf735c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/7697c63f5e5243bc855b5bfd8fcf735c 2024-11-17T07:14:32,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5a2d373fe61941d2a9d19a2e7c5c37e4 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/5a2d373fe61941d2a9d19a2e7c5c37e4 2024-11-17T07:14:32,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/8a4fd0ebf14540b29c6918e44e83a181 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/8a4fd0ebf14540b29c6918e44e83a181 2024-11-17T07:14:32,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/2fc5659a759f438b8cc13c64c85cbaef to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/2fc5659a759f438b8cc13c64c85cbaef 2024-11-17T07:14:32,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/2ce36032795e4f92836349496d87c9a9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/c752fdfd6d5f414dbd3a54f9f7a652c4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/ff8fa00b9eb04bfba6c6302309b49194, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/9a816cc616d8414fad92fdf0431b15c3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/83f5157c9272425cba4ce373694db4b6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/29cdea3e06f94297b492fdc1c8acf32c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/f320de309ab547cc9d2aa592880deac5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/a4742ee4836e4865bf26f37d150f13c8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/2dcfa9a83daa4fbaa29d3db56d9faa52, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/4ba7f39a9ae648c9b61be1f774b52a57, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/ba3361666fb54233892f1a5614b0aa54, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/80a4a6b817d04beaa4ce6bffa8faff6b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/dbb480108a934ee8b57971fb3e11ab93, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/6faf0b6b807e4b24b6271bee599c208a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/6a6898f02f584b39ab1b5ae840710730, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/533036d0df3a4ca39d57cd42ccec663d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/f1fc88b9d2444f72aa5cb30334f205e1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/9ad7c702bb0441d987e10664295403d8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/97176c6783a0441ba412adb169603940, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/98271dc6c59e43dc9ffb16a24dd152ab, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/544048d34fbb488ca9938a63621b7443, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/70b48cb1397344c6984bc97498880dcc, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/c77a147f372344578b60e9e94ed38a52, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/703ee3e078b44fe3aa615a4ccfa1492b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/32403413fc8349a2bbc3298445882ce8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/8b2c0a7d01484d6ba2ca9a9144c88191, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/d740302d54fd4281ae289b3790c9faee] to archive 2024-11-17T07:14:32,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:14:32,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/2ce36032795e4f92836349496d87c9a9 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/2ce36032795e4f92836349496d87c9a9 2024-11-17T07:14:32,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/c752fdfd6d5f414dbd3a54f9f7a652c4 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/c752fdfd6d5f414dbd3a54f9f7a652c4 2024-11-17T07:14:32,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/ff8fa00b9eb04bfba6c6302309b49194 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/ff8fa00b9eb04bfba6c6302309b49194 2024-11-17T07:14:32,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/9a816cc616d8414fad92fdf0431b15c3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/9a816cc616d8414fad92fdf0431b15c3 2024-11-17T07:14:32,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/83f5157c9272425cba4ce373694db4b6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/83f5157c9272425cba4ce373694db4b6 2024-11-17T07:14:32,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/29cdea3e06f94297b492fdc1c8acf32c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/29cdea3e06f94297b492fdc1c8acf32c 2024-11-17T07:14:32,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/f320de309ab547cc9d2aa592880deac5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/f320de309ab547cc9d2aa592880deac5 2024-11-17T07:14:32,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/a4742ee4836e4865bf26f37d150f13c8 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/a4742ee4836e4865bf26f37d150f13c8 2024-11-17T07:14:32,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/2dcfa9a83daa4fbaa29d3db56d9faa52 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/2dcfa9a83daa4fbaa29d3db56d9faa52 2024-11-17T07:14:32,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/4ba7f39a9ae648c9b61be1f774b52a57 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/4ba7f39a9ae648c9b61be1f774b52a57 2024-11-17T07:14:32,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/ba3361666fb54233892f1a5614b0aa54 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/ba3361666fb54233892f1a5614b0aa54 2024-11-17T07:14:32,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/80a4a6b817d04beaa4ce6bffa8faff6b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/80a4a6b817d04beaa4ce6bffa8faff6b 2024-11-17T07:14:32,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/dbb480108a934ee8b57971fb3e11ab93 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/dbb480108a934ee8b57971fb3e11ab93 2024-11-17T07:14:32,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/6faf0b6b807e4b24b6271bee599c208a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/6faf0b6b807e4b24b6271bee599c208a 2024-11-17T07:14:32,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/6a6898f02f584b39ab1b5ae840710730 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/6a6898f02f584b39ab1b5ae840710730 2024-11-17T07:14:32,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/533036d0df3a4ca39d57cd42ccec663d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/533036d0df3a4ca39d57cd42ccec663d 2024-11-17T07:14:32,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/f1fc88b9d2444f72aa5cb30334f205e1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/f1fc88b9d2444f72aa5cb30334f205e1 2024-11-17T07:14:32,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/9ad7c702bb0441d987e10664295403d8 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/9ad7c702bb0441d987e10664295403d8 2024-11-17T07:14:32,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/97176c6783a0441ba412adb169603940 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/97176c6783a0441ba412adb169603940 2024-11-17T07:14:32,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/98271dc6c59e43dc9ffb16a24dd152ab to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/98271dc6c59e43dc9ffb16a24dd152ab 2024-11-17T07:14:32,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/544048d34fbb488ca9938a63621b7443 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/544048d34fbb488ca9938a63621b7443 2024-11-17T07:14:32,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/70b48cb1397344c6984bc97498880dcc to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/70b48cb1397344c6984bc97498880dcc 2024-11-17T07:14:32,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/c77a147f372344578b60e9e94ed38a52 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/c77a147f372344578b60e9e94ed38a52 2024-11-17T07:14:32,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/703ee3e078b44fe3aa615a4ccfa1492b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/703ee3e078b44fe3aa615a4ccfa1492b 2024-11-17T07:14:32,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/32403413fc8349a2bbc3298445882ce8 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/32403413fc8349a2bbc3298445882ce8 2024-11-17T07:14:32,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/8b2c0a7d01484d6ba2ca9a9144c88191 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/8b2c0a7d01484d6ba2ca9a9144c88191 2024-11-17T07:14:32,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/d740302d54fd4281ae289b3790c9faee to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/d740302d54fd4281ae289b3790c9faee 2024-11-17T07:14:32,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/96baeb837c3047c789aa2b69b1525e69, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/ad2fab4db0be44a9bcc6a87dbb12b0d8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d3d73b550c7e4911b730e958f4a29e18, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/a316ddaf691b4e338aa4db331a22ce9c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d0dfd12e8897460e88719ae7be2058b9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/e69f4182bde44f238d6025fa4477aab6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d8c916e744674852970fbb493c654efa, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/5edaf5b6b0564f4e8abbc76345f6bbc9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/1c96725d310d41e38ecdc9fe615d877a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/a338e848771d4089abb5401a7b0d4e90, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/52c45476eeed42e8b1da95a1f32a8b52, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/f12a3f39477242dca103241c1e779108, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/cc9d18c416c84a47bfc596cd25e8f4b2, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/fb1e0bb0907840de95de8d28cfd35b4e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/68a48e8e55424ebc9b6361a5edcb4828, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/697af042c7914dda994ebe1ceaf5ea25, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/b1aaa98de1fe494dbeb4f8d138f9e67f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/777785a4ba9d4d579c95ba1bd1c503a2, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/2f9f7b9f2b41435b984a01b097e2b171, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/4b2bb09e92fd44e7ae11c3cc7a075c32, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/630531e43a84422ebb28b522896fe7dd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/13bdbe4e8ebb47b4ae5a1bd0a7112dcc, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/ca292d2060f24d56b80cccfc48b1104e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/8ff790bd48114358a24a6a1c703076c1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/8a97a04551954d1b942241aa9750f22a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/82f966c6ce7c4fe0a3f40f84fede16dc, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/2926492b21b84dbd9ecca4a79b72c9aa] to archive 2024-11-17T07:14:32,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:14:32,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/96baeb837c3047c789aa2b69b1525e69 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/96baeb837c3047c789aa2b69b1525e69 2024-11-17T07:14:32,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/ad2fab4db0be44a9bcc6a87dbb12b0d8 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/ad2fab4db0be44a9bcc6a87dbb12b0d8 2024-11-17T07:14:32,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d3d73b550c7e4911b730e958f4a29e18 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d3d73b550c7e4911b730e958f4a29e18 2024-11-17T07:14:32,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/a316ddaf691b4e338aa4db331a22ce9c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/a316ddaf691b4e338aa4db331a22ce9c 2024-11-17T07:14:32,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d0dfd12e8897460e88719ae7be2058b9 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d0dfd12e8897460e88719ae7be2058b9 2024-11-17T07:14:32,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/e69f4182bde44f238d6025fa4477aab6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/e69f4182bde44f238d6025fa4477aab6 2024-11-17T07:14:32,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d8c916e744674852970fbb493c654efa to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/d8c916e744674852970fbb493c654efa 2024-11-17T07:14:32,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/5edaf5b6b0564f4e8abbc76345f6bbc9 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/5edaf5b6b0564f4e8abbc76345f6bbc9 2024-11-17T07:14:32,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/1c96725d310d41e38ecdc9fe615d877a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/1c96725d310d41e38ecdc9fe615d877a 2024-11-17T07:14:32,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/a338e848771d4089abb5401a7b0d4e90 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/a338e848771d4089abb5401a7b0d4e90 2024-11-17T07:14:32,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/52c45476eeed42e8b1da95a1f32a8b52 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/52c45476eeed42e8b1da95a1f32a8b52 2024-11-17T07:14:32,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/f12a3f39477242dca103241c1e779108 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/f12a3f39477242dca103241c1e779108 2024-11-17T07:14:32,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/cc9d18c416c84a47bfc596cd25e8f4b2 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/cc9d18c416c84a47bfc596cd25e8f4b2 2024-11-17T07:14:32,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/fb1e0bb0907840de95de8d28cfd35b4e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/fb1e0bb0907840de95de8d28cfd35b4e 2024-11-17T07:14:32,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/68a48e8e55424ebc9b6361a5edcb4828 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/68a48e8e55424ebc9b6361a5edcb4828 2024-11-17T07:14:32,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/697af042c7914dda994ebe1ceaf5ea25 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/697af042c7914dda994ebe1ceaf5ea25 2024-11-17T07:14:32,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/b1aaa98de1fe494dbeb4f8d138f9e67f to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/b1aaa98de1fe494dbeb4f8d138f9e67f 2024-11-17T07:14:32,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/777785a4ba9d4d579c95ba1bd1c503a2 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/777785a4ba9d4d579c95ba1bd1c503a2 2024-11-17T07:14:32,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/2f9f7b9f2b41435b984a01b097e2b171 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/2f9f7b9f2b41435b984a01b097e2b171 2024-11-17T07:14:32,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/4b2bb09e92fd44e7ae11c3cc7a075c32 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/4b2bb09e92fd44e7ae11c3cc7a075c32 2024-11-17T07:14:32,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/630531e43a84422ebb28b522896fe7dd to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/630531e43a84422ebb28b522896fe7dd 2024-11-17T07:14:32,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/13bdbe4e8ebb47b4ae5a1bd0a7112dcc to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/13bdbe4e8ebb47b4ae5a1bd0a7112dcc 2024-11-17T07:14:32,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/ca292d2060f24d56b80cccfc48b1104e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/ca292d2060f24d56b80cccfc48b1104e 2024-11-17T07:14:32,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/8ff790bd48114358a24a6a1c703076c1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/8ff790bd48114358a24a6a1c703076c1 2024-11-17T07:14:32,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/8a97a04551954d1b942241aa9750f22a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/8a97a04551954d1b942241aa9750f22a 2024-11-17T07:14:32,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/82f966c6ce7c4fe0a3f40f84fede16dc to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/82f966c6ce7c4fe0a3f40f84fede16dc 2024-11-17T07:14:32,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/2926492b21b84dbd9ecca4a79b72c9aa to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/2926492b21b84dbd9ecca4a79b72c9aa 2024-11-17T07:14:32,925 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/recovered.edits/421.seqid, newMaxSeqId=421, maxSeqId=4 2024-11-17T07:14:32,926 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. 2024-11-17T07:14:32,926 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for efcef908f9da2d14e10a65888eb3eb6d: Waiting for close lock at 1731827671171Running coprocessor pre-close hooks at 1731827671171Disabling compacts and flushes for region at 1731827671171Disabling writes for close at 1731827671171Obtaining lock to block concurrent updates at 1731827671171Preparing flush snapshotting stores in efcef908f9da2d14e10a65888eb3eb6d at 1731827671171Finished memstore snapshotting TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731827671172 (+1 ms)Flushing stores of TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d. at 1731827671172Flushing efcef908f9da2d14e10a65888eb3eb6d/A: creating writer at 1731827671172Flushing mob file efcef908f9da2d14e10a65888eb3eb6d/A: appending metadata at 1731827671177 (+5 ms)Flushing mob file efcef908f9da2d14e10a65888eb3eb6d/A: closing flushed file at 1731827671177Flushing efcef908f9da2d14e10a65888eb3eb6d/A: appending metadata at 1731827671586 (+409 ms)Flushing efcef908f9da2d14e10a65888eb3eb6d/A: closing flushed file at 1731827671586Flushing efcef908f9da2d14e10a65888eb3eb6d/B: creating writer at 1731827671995 (+409 ms)Flushing efcef908f9da2d14e10a65888eb3eb6d/B: appending metadata at 1731827671998 (+3 ms)Flushing efcef908f9da2d14e10a65888eb3eb6d/B: closing flushed file at 1731827671998Flushing efcef908f9da2d14e10a65888eb3eb6d/C: creating writer at 1731827672406 (+408 ms)Flushing efcef908f9da2d14e10a65888eb3eb6d/C: appending metadata at 1731827672409 (+3 ms)Flushing efcef908f9da2d14e10a65888eb3eb6d/C: closing flushed file at 1731827672409Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63dedb9a: reopening flushed file at 1731827672818 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bf47a43: reopening flushed file at 1731827672822 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54c65463: reopening flushed file at 1731827672826 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for efcef908f9da2d14e10a65888eb3eb6d in 1659ms, sequenceid=418, compaction requested=true at 1731827672830 (+4 ms)Writing region close event to WAL at 1731827672922 (+92 ms)Running coprocessor post-close hooks at 1731827672926 (+4 ms)Closed at 1731827672926 2024-11-17T07:14:32,928 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:32,929 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=efcef908f9da2d14e10a65888eb3eb6d, regionState=CLOSED 2024-11-17T07:14:32,930 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:14:32,931 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T07:14:32,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-17T07:14:32,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure efcef908f9da2d14e10a65888eb3eb6d, server=1dd62e24b303,36419,1731827614816 in 1.9120 sec 2024-11-17T07:14:32,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-17T07:14:32,934 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=efcef908f9da2d14e10a65888eb3eb6d, UNASSIGN in 1.9160 sec 2024-11-17T07:14:32,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-17T07:14:32,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9210 sec 2024-11-17T07:14:32,937 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827672937"}]},"ts":"1731827672937"} 2024-11-17T07:14:32,938 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T07:14:32,938 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T07:14:32,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9290 sec 2024-11-17T07:14:33,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-17T07:14:33,157 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:33,158 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T07:14:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:33,159 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T07:14:33,160 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:33,162 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,164 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/recovered.edits] 2024-11-17T07:14:33,167 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/47f6c1436e7d4baab05d5cc0bfbefeba to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/47f6c1436e7d4baab05d5cc0bfbefeba 2024-11-17T07:14:33,168 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/60f2904c3fa54c21beb301191ff3f815 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/60f2904c3fa54c21beb301191ff3f815 2024-11-17T07:14:33,169 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/9167257199494169a81c4178ae7bc9b0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/A/9167257199494169a81c4178ae7bc9b0 2024-11-17T07:14:33,171 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/02b54bbad3db435aba16794edd5881e6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/02b54bbad3db435aba16794edd5881e6 2024-11-17T07:14:33,172 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/d28b0cb92404457d9a83107415d3548d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/d28b0cb92404457d9a83107415d3548d 2024-11-17T07:14:33,173 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/fd7302eba0a949cfaa977e5eb3b0c88e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/B/fd7302eba0a949cfaa977e5eb3b0c88e 2024-11-17T07:14:33,176 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/18f98b80ac004f8eb8491c185296aaac to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/18f98b80ac004f8eb8491c185296aaac 2024-11-17T07:14:33,177 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/61693a32479643418692748182fe137a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/61693a32479643418692748182fe137a 2024-11-17T07:14:33,178 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/6feb0fc6495547b7b4e65699757b1f69 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/C/6feb0fc6495547b7b4e65699757b1f69 2024-11-17T07:14:33,182 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/recovered.edits/421.seqid to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d/recovered.edits/421.seqid 2024-11-17T07:14:33,183 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,183 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T07:14:33,184 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T07:14:33,184 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T07:14:33,188 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170396c3ba483d473e8e59eb0dd960ca90_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170396c3ba483d473e8e59eb0dd960ca90_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,189 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117100a28ed256a4c7cacfb1a54d1e8f05c_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117100a28ed256a4c7cacfb1a54d1e8f05c_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,190 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117226a868d79c844fd8fa43d7312d9145a_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117226a868d79c844fd8fa43d7312d9145a_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,191 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117254fe249f5db4af78b27c4c3fa882f7d_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117254fe249f5db4af78b27c4c3fa882f7d_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,192 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174682e530b6df49179b730ce7bb69039a_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174682e530b6df49179b730ce7bb69039a_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,193 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111753abfa2330c74cbdb0f9288731412149_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111753abfa2330c74cbdb0f9288731412149_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,194 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175db2fcbc77904faeabaf5cbe5c421b41_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411175db2fcbc77904faeabaf5cbe5c421b41_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,195 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177cee5f4177964578945543a51e5b7b1a_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411177cee5f4177964578945543a51e5b7b1a_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,196 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111780da8e5a621240e2b6b229581b95b746_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111780da8e5a621240e2b6b229581b95b746_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,197 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117846e95b59efb439a90050cbad96902d7_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117846e95b59efb439a90050cbad96902d7_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,198 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179a8e765332914f1fa3df4d74c6dc64af_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179a8e765332914f1fa3df4d74c6dc64af_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,199 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117aec8f495240745d5a6abd9e4fddb3bb2_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117aec8f495240745d5a6abd9e4fddb3bb2_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,200 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bb2938e072824191bcdf2b579215dfad_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117bb2938e072824191bcdf2b579215dfad_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,201 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c17f040639384eb5b05146167a9f7ffc_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c17f040639384eb5b05146167a9f7ffc_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,202 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ccf31bdb7def405facf61381430b9805_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ccf31bdb7def405facf61381430b9805_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,203 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ce2a53148b28475f968f6a063da887f7_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ce2a53148b28475f968f6a063da887f7_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,204 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d2a499c48eba45a386e79121dc735687_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d2a499c48eba45a386e79121dc735687_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,205 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d327cd31085848799bd02009e3dc0b1a_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d327cd31085848799bd02009e3dc0b1a_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,206 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d938dea7b6e14297b1f1696a3d394193_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d938dea7b6e14297b1f1696a3d394193_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,207 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117db2cf67578f6484d8e44b4fdb335c6ea_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117db2cf67578f6484d8e44b4fdb335c6ea_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,208 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ef19b615ef23420996cfb2f56ac2eb3f_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ef19b615ef23420996cfb2f56ac2eb3f_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,209 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f77f030eae1b4b6f840a45734ae45ffa_efcef908f9da2d14e10a65888eb3eb6d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f77f030eae1b4b6f840a45734ae45ffa_efcef908f9da2d14e10a65888eb3eb6d 2024-11-17T07:14:33,209 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T07:14:33,211 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:33,213 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T07:14:33,215 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T07:14:33,216 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:33,216 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T07:14:33,217 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731827673216"}]},"ts":"9223372036854775807"} 2024-11-17T07:14:33,219 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T07:14:33,219 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => efcef908f9da2d14e10a65888eb3eb6d, NAME => 'TestAcidGuarantees,,1731827645118.efcef908f9da2d14e10a65888eb3eb6d.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T07:14:33,219 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T07:14:33,219 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731827673219"}]},"ts":"9223372036854775807"} 2024-11-17T07:14:33,220 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T07:14:33,221 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:33,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 63 msec 2024-11-17T07:14:33,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-17T07:14:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T07:14:33,270 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:33,277 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 249) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/cluster_ec49d093-dc1d-39e4-b51d-2529c25f0961/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-49537162_22 at /127.0.0.1:33740 [Waiting for operation #1003] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/cluster_ec49d093-dc1d-39e4-b51d-2529c25f0961/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_411634984_22 at /127.0.0.1:45732 [Waiting for operation #1002] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-49537162_22 at /127.0.0.1:46990 [Waiting for operation #538] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_411634984_22 at /127.0.0.1:47004 [Waiting for operation #522] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=436 (was 419) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1756 (was 2242) 2024-11-17T07:14:33,286 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=436, ProcessCount=11, AvailableMemoryMB=1756 2024-11-17T07:14:33,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T07:14:33,287 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T07:14:33,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:33,289 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T07:14:33,289 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:33,289 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-17T07:14:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-17T07:14:33,290 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T07:14:33,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742080_1256 (size=1061) 2024-11-17T07:14:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-17T07:14:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-17T07:14:33,698 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650 2024-11-17T07:14:33,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742081_1257 (size=53) 2024-11-17T07:14:33,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-17T07:14:34,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:14:34,104 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6d8c50823ce967d5944d0259cb160d17, disabling compactions & flushes 2024-11-17T07:14:34,104 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:34,104 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:34,104 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. after waiting 0 ms 2024-11-17T07:14:34,104 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:34,104 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:34,104 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6d8c50823ce967d5944d0259cb160d17: Waiting for close lock at 1731827674104Disabling compacts and flushes for region at 1731827674104Disabling writes for close at 1731827674104Writing region close event to WAL at 1731827674104Closed at 1731827674104 2024-11-17T07:14:34,105 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T07:14:34,105 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731827674105"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731827674105"}]},"ts":"1731827674105"} 2024-11-17T07:14:34,106 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T07:14:34,107 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T07:14:34,107 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827674107"}]},"ts":"1731827674107"} 2024-11-17T07:14:34,108 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T07:14:34,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d8c50823ce967d5944d0259cb160d17, ASSIGN}] 2024-11-17T07:14:34,109 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d8c50823ce967d5944d0259cb160d17, ASSIGN 2024-11-17T07:14:34,110 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d8c50823ce967d5944d0259cb160d17, ASSIGN; state=OFFLINE, location=1dd62e24b303,36419,1731827614816; forceNewPlan=false, retain=false 2024-11-17T07:14:34,261 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=6d8c50823ce967d5944d0259cb160d17, regionState=OPENING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:34,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d8c50823ce967d5944d0259cb160d17, ASSIGN because future has completed 2024-11-17T07:14:34,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:14:34,388 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T07:14:34,417 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:34,417 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} 2024-11-17T07:14:34,417 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:34,418 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:14:34,418 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:34,418 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:34,419 INFO [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:34,420 INFO [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:14:34,420 INFO [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d8c50823ce967d5944d0259cb160d17 columnFamilyName A 2024-11-17T07:14:34,420 DEBUG [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:34,421 INFO [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] regionserver.HStore(327): Store=6d8c50823ce967d5944d0259cb160d17/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:14:34,421 INFO [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:34,422 INFO [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:14:34,422 INFO [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d8c50823ce967d5944d0259cb160d17 columnFamilyName B 2024-11-17T07:14:34,422 DEBUG [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:34,422 INFO [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] regionserver.HStore(327): Store=6d8c50823ce967d5944d0259cb160d17/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:14:34,422 INFO [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:34,423 INFO [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:14:34,423 INFO [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d8c50823ce967d5944d0259cb160d17 columnFamilyName C 2024-11-17T07:14:34,423 DEBUG [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:14:34,423 INFO [StoreOpener-6d8c50823ce967d5944d0259cb160d17-1 {}] regionserver.HStore(327): Store=6d8c50823ce967d5944d0259cb160d17/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:14:34,424 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:34,424 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:34,424 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:34,425 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:34,425 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:34,425 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:34,426 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T07:14:34,427 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:34,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-17T07:14:34,428 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T07:14:34,428 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 6d8c50823ce967d5944d0259cb160d17; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64902256, jitterRate=-0.03288102149963379}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T07:14:34,429 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:34,429 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 6d8c50823ce967d5944d0259cb160d17: Running coprocessor pre-open hook at 1731827674418Writing region info on filesystem at 1731827674418Initializing all the Stores at 1731827674418Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827674418Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827674419 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827674419Cleaning up temporary data from old regions at 1731827674425 (+6 ms)Running coprocessor post-open hooks at 1731827674429 (+4 ms)Region opened successfully at 1731827674429 2024-11-17T07:14:34,430 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., pid=63, masterSystemTime=1731827674414 2024-11-17T07:14:34,431 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:34,431 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:34,432 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=6d8c50823ce967d5944d0259cb160d17, regionState=OPEN, openSeqNum=2, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:34,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:14:34,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-17T07:14:34,435 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 in 172 msec 2024-11-17T07:14:34,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-17T07:14:34,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d8c50823ce967d5944d0259cb160d17, ASSIGN in 326 msec 2024-11-17T07:14:34,437 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T07:14:34,437 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827674437"}]},"ts":"1731827674437"} 2024-11-17T07:14:34,438 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T07:14:34,439 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T07:14:34,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-17T07:14:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-17T07:14:35,437 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:35,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@333e91c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:35,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:35,440 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eebc365, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:35,441 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:35,442 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:35,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1db4879f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,443 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:35,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:35,445 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:35,446 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T07:14:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 1dd62e24b303,36491,1731827613877 2024-11-17T07:14:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4add4f9a 2024-11-17T07:14:35,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T07:14:35,447 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T07:14:35,449 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25146acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,449 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:35,449 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:35,449 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f003170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:35,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:35,451 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:35,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba072be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,452 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@391cff66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,452 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:35,452 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:35,452 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746f89a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:35,453 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:35,453 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d113fbf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,454 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@135bd0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,454 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:35,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:35,455 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77501b39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:35,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:35,456 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:35,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40422e6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4609cead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,457 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:35,457 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:35,458 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17b7cdb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:35,459 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5320feea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a2df097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,460 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:35,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:35,461 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53b34ce6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:35,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:35,462 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:35,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3db7320d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5559acf0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,463 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:35,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:35,464 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:35,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:35,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:35,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@632c0aa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:35,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:35,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:35,465 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:35,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@613fa4c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,465 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a4f6020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,465 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:35,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:35,466 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc4352e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:35,467 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77f4305f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692cba37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:35,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:35,469 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@778c3b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:35,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:35,470 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:35,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3394db16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d805c88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,471 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:35,471 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:35,471 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:35,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526e3409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:35,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:35,473 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:35,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3da43dd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4861c2d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:14:35,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:14:35,474 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:14:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:14:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:14:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8d2c665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:14:35,474 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:14:35,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:35,475 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:14:35,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b71ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:14:35,476 DEBUG [Thread-1185 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:35,476 DEBUG [Thread-1189 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:35,476 DEBUG [Thread-1196 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:35,476 DEBUG [Thread-1198 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:35,476 DEBUG [Thread-1200 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:35,476 DEBUG [Thread-1204 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:35,477 DEBUG [Thread-1202 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:35,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:35,477 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:35,478 DEBUG [Thread-1198 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:35,478 DEBUG [Thread-1198 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:35,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:35,478 DEBUG [Thread-1187 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-17T07:14:35,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:35,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:35,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:35,479 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:35,479 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:35,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:35,480 DEBUG [Thread-1202 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:35,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T07:14:35,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:35,480 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:35,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:35,481 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:35,481 DEBUG [Thread-1193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:35,481 DEBUG [Thread-1191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:14:35,481 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:35,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:14:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:14:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:14:35,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:35,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:35,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:35,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:14:35,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:14:35,484 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:14:35,485 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:14:35,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:14:35,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:14:35,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:14:35,488 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:14:35,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:14:35,490 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:14:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:35,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:14:35,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:35,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:35,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:35,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:35,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:35,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:35,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/ef429386e49843eba6b6532ce81db6c5 is 50, key is test_row_0/A:col10/1731827675496/Put/seqid=0 2024-11-17T07:14:35,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742082_1258 (size=12001) 2024-11-17T07:14:35,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827675568, max timestamp=1731827675579Num uniques 859; 2024-11-17T07:14:35,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:35,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827675568, max timestamp=1731827675581Num uniques 859; 2024-11-17T07:14:35,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:35,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827675568, max timestamp=1731827675581Num uniques 859; 2024-11-17T07:14:35,584 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:35,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T07:14:35,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827735616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827735616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827735623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827735623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827735625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T07:14:35,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:35,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:35,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:35,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:35,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:35,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827735737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827735737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827735737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827735737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827735737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T07:14:35,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:35,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:35,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:35,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:35,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:35,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:35,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T07:14:35,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/ef429386e49843eba6b6532ce81db6c5 2024-11-17T07:14:35,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T07:14:35,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:35,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:35,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:35,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:35,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:35,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:35,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827735947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827735947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827735947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827735947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:35,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827735947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:35,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:35,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:35,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/0b7914197e654db882ecb46ec41f2420 is 50, key is test_row_0/B:col10/1731827675496/Put/seqid=0 2024-11-17T07:14:35,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742083_1259 (size=12001) 2024-11-17T07:14:35,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/0b7914197e654db882ecb46ec41f2420 2024-11-17T07:14:36,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/14addfc3568d4a1691424c71c49a6019 is 50, key is test_row_0/C:col10/1731827675496/Put/seqid=0 2024-11-17T07:14:36,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742084_1260 (size=12001) 2024-11-17T07:14:36,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/14addfc3568d4a1691424c71c49a6019 2024-11-17T07:14:36,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/ef429386e49843eba6b6532ce81db6c5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/ef429386e49843eba6b6532ce81db6c5 2024-11-17T07:14:36,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/ef429386e49843eba6b6532ce81db6c5, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T07:14:36,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/0b7914197e654db882ecb46ec41f2420 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/0b7914197e654db882ecb46ec41f2420 2024-11-17T07:14:36,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/0b7914197e654db882ecb46ec41f2420, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T07:14:36,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/14addfc3568d4a1691424c71c49a6019 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/14addfc3568d4a1691424c71c49a6019 2024-11-17T07:14:36,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/14addfc3568d4a1691424c71c49a6019, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T07:14:36,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6d8c50823ce967d5944d0259cb160d17 in 539ms, sequenceid=12, compaction requested=false 2024-11-17T07:14:36,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:36,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-17T07:14:36,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:36,100 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-17T07:14:36,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:36,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:36,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:36,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:36,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:36,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:36,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/d41dfe8443964a33b112f3ebbdf86b87 is 50, key is test_row_0/A:col10/1731827675623/Put/seqid=0 2024-11-17T07:14:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T07:14:36,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742085_1261 (size=12001) 2024-11-17T07:14:36,113 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/d41dfe8443964a33b112f3ebbdf86b87 2024-11-17T07:14:36,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/2c245a1c4ff9485f9853c8b9a9ee94db is 50, key is test_row_0/B:col10/1731827675623/Put/seqid=0 2024-11-17T07:14:36,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742086_1262 (size=12001) 2024-11-17T07:14:36,129 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/2c245a1c4ff9485f9853c8b9a9ee94db 2024-11-17T07:14:36,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/5da112d639a841f8aff096330cb3acd6 is 50, key is test_row_0/C:col10/1731827675623/Put/seqid=0 2024-11-17T07:14:36,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742087_1263 (size=12001) 2024-11-17T07:14:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:36,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:36,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827736263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827736264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827736268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827736269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,273 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827736270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827736377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827736377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,382 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827736377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827736377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827736378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,588 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/5da112d639a841f8aff096330cb3acd6 2024-11-17T07:14:36,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827736587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827736587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827736588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827736588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,592 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827736588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/d41dfe8443964a33b112f3ebbdf86b87 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/d41dfe8443964a33b112f3ebbdf86b87 2024-11-17T07:14:36,597 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/d41dfe8443964a33b112f3ebbdf86b87, entries=150, sequenceid=48, filesize=11.7 K 2024-11-17T07:14:36,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/2c245a1c4ff9485f9853c8b9a9ee94db as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2c245a1c4ff9485f9853c8b9a9ee94db 2024-11-17T07:14:36,603 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2c245a1c4ff9485f9853c8b9a9ee94db, entries=150, sequenceid=48, filesize=11.7 K 2024-11-17T07:14:36,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/5da112d639a841f8aff096330cb3acd6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5da112d639a841f8aff096330cb3acd6 2024-11-17T07:14:36,608 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5da112d639a841f8aff096330cb3acd6, entries=150, sequenceid=48, filesize=11.7 K 2024-11-17T07:14:36,609 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 6d8c50823ce967d5944d0259cb160d17 in 509ms, sequenceid=48, compaction requested=false 2024-11-17T07:14:36,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:36,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:36,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-17T07:14:36,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-17T07:14:36,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-17T07:14:36,613 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1300 sec 2024-11-17T07:14:36,615 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.1370 sec 2024-11-17T07:14:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-17T07:14:36,618 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:36,618 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-17T07:14:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T07:14:36,620 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:36,621 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:36,621 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T07:14:36,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-17T07:14:36,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:36,774 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:14:36,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:36,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:36,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:36,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:36,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:36,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:36,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/1f1f6d5b5c6146b282904d945adb2a3a is 50, key is test_row_0/A:col10/1731827676268/Put/seqid=0 2024-11-17T07:14:36,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742088_1264 (size=12001) 2024-11-17T07:14:36,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:36,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:36,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827736916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,923 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827676915, max timestamp=1731827676921Num uniques 859; 2024-11-17T07:14:36,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827676915, max timestamp=1731827676921Num uniques 859; 2024-11-17T07:14:36,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:36,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:36,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827676915, max timestamp=1731827676921Num uniques 859; 2024-11-17T07:14:36,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:36,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T07:14:36,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827736945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827736945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827736950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827736951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:36,957 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827737027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827737057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827737057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827737067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827737067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,072 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,184 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/1f1f6d5b5c6146b282904d945adb2a3a 2024-11-17T07:14:37,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/dd9264361de04c49a200cf1332ad2026 is 50, key is test_row_0/B:col10/1731827676268/Put/seqid=0 2024-11-17T07:14:37,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742089_1265 (size=12001) 2024-11-17T07:14:37,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827737237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T07:14:37,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827737267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827737267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,272 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827737277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827737277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,282 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827737547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827737578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827737578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827737587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:37,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827737587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:37,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:37,597 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/dd9264361de04c49a200cf1332ad2026 2024-11-17T07:14:37,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/d764b5aa4d8048baa37cdb0b1e3cc582 is 50, key is test_row_0/C:col10/1731827676268/Put/seqid=0 2024-11-17T07:14:37,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742090_1266 (size=12001) 2024-11-17T07:14:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T07:14:38,009 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/d764b5aa4d8048baa37cdb0b1e3cc582 2024-11-17T07:14:38,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/1f1f6d5b5c6146b282904d945adb2a3a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1f1f6d5b5c6146b282904d945adb2a3a 2024-11-17T07:14:38,018 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1f1f6d5b5c6146b282904d945adb2a3a, entries=150, sequenceid=59, filesize=11.7 K 2024-11-17T07:14:38,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/dd9264361de04c49a200cf1332ad2026 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/dd9264361de04c49a200cf1332ad2026 2024-11-17T07:14:38,022 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/dd9264361de04c49a200cf1332ad2026, entries=150, sequenceid=59, filesize=11.7 K 2024-11-17T07:14:38,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/d764b5aa4d8048baa37cdb0b1e3cc582 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d764b5aa4d8048baa37cdb0b1e3cc582 2024-11-17T07:14:38,027 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d764b5aa4d8048baa37cdb0b1e3cc582, entries=150, sequenceid=59, filesize=11.7 K 2024-11-17T07:14:38,028 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6d8c50823ce967d5944d0259cb160d17 in 1254ms, sequenceid=59, compaction requested=true 2024-11-17T07:14:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:38,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-17T07:14:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-17T07:14:38,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-17T07:14:38,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-17T07:14:38,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.4140 sec 2024-11-17T07:14:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:38,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-17T07:14:38,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:38,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:38,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:38,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:38,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:38,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:38,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/5c58245a9a87446f98280b1f34b02bfa is 50, key is test_row_0/A:col10/1731827676943/Put/seqid=0 2024-11-17T07:14:38,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742091_1267 (size=14341) 2024-11-17T07:14:38,070 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/5c58245a9a87446f98280b1f34b02bfa 2024-11-17T07:14:38,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/43d12f86aaa2470da60cc5ae1785b375 is 50, key is test_row_0/B:col10/1731827676943/Put/seqid=0 2024-11-17T07:14:38,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742092_1268 (size=12001) 2024-11-17T07:14:38,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827738093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827738094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827738095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827738097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827738097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827738207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827738207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827738207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,212 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827738417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827738417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827738418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,447 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T07:14:38,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/43d12f86aaa2470da60cc5ae1785b375 2024-11-17T07:14:38,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/ca1a58b9920d4a62865cd48f4d50635a is 50, key is test_row_0/C:col10/1731827676943/Put/seqid=0 2024-11-17T07:14:38,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742093_1269 (size=12001) 2024-11-17T07:14:38,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/ca1a58b9920d4a62865cd48f4d50635a 2024-11-17T07:14:38,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/5c58245a9a87446f98280b1f34b02bfa as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5c58245a9a87446f98280b1f34b02bfa 2024-11-17T07:14:38,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5c58245a9a87446f98280b1f34b02bfa, entries=200, sequenceid=96, filesize=14.0 K 2024-11-17T07:14:38,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/43d12f86aaa2470da60cc5ae1785b375 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/43d12f86aaa2470da60cc5ae1785b375 2024-11-17T07:14:38,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/43d12f86aaa2470da60cc5ae1785b375, entries=150, sequenceid=96, filesize=11.7 K 2024-11-17T07:14:38,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/ca1a58b9920d4a62865cd48f4d50635a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/ca1a58b9920d4a62865cd48f4d50635a 2024-11-17T07:14:38,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/ca1a58b9920d4a62865cd48f4d50635a, entries=150, sequenceid=96, filesize=11.7 K 2024-11-17T07:14:38,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 6d8c50823ce967d5944d0259cb160d17 in 462ms, sequenceid=96, compaction requested=true 2024-11-17T07:14:38,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:38,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:38,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:38,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:38,524 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:38,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:38,524 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:38,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:38,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:38,525 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:38,525 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/B is initiating minor compaction (all files) 2024-11-17T07:14:38,525 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/B in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:38,526 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/0b7914197e654db882ecb46ec41f2420, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2c245a1c4ff9485f9853c8b9a9ee94db, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/dd9264361de04c49a200cf1332ad2026, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/43d12f86aaa2470da60cc5ae1785b375] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=46.9 K 2024-11-17T07:14:38,526 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:38,526 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/A is initiating minor compaction (all files) 2024-11-17T07:14:38,526 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/A in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:38,526 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/ef429386e49843eba6b6532ce81db6c5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/d41dfe8443964a33b112f3ebbdf86b87, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1f1f6d5b5c6146b282904d945adb2a3a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5c58245a9a87446f98280b1f34b02bfa] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=49.2 K 2024-11-17T07:14:38,526 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b7914197e654db882ecb46ec41f2420, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731827675489 2024-11-17T07:14:38,526 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef429386e49843eba6b6532ce81db6c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731827675489 2024-11-17T07:14:38,527 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting d41dfe8443964a33b112f3ebbdf86b87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731827675612 2024-11-17T07:14:38,527 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c245a1c4ff9485f9853c8b9a9ee94db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731827675612 2024-11-17T07:14:38,527 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f1f6d5b5c6146b282904d945adb2a3a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731827676262 2024-11-17T07:14:38,527 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting dd9264361de04c49a200cf1332ad2026, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731827676262 2024-11-17T07:14:38,527 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c58245a9a87446f98280b1f34b02bfa, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731827676943 2024-11-17T07:14:38,528 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 43d12f86aaa2470da60cc5ae1785b375, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731827676943 2024-11-17T07:14:38,537 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:38,537 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#B#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:38,537 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/b715ce85a4cc43b79845a79cae95cfe7 is 50, key is test_row_0/A:col10/1731827676943/Put/seqid=0 2024-11-17T07:14:38,538 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/326a859ee39d4dee80c11417199ade3d is 50, key is test_row_0/B:col10/1731827676943/Put/seqid=0 2024-11-17T07:14:38,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742094_1270 (size=12139) 2024-11-17T07:14:38,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742095_1271 (size=12139) 2024-11-17T07:14:38,554 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/b715ce85a4cc43b79845a79cae95cfe7 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/b715ce85a4cc43b79845a79cae95cfe7 2024-11-17T07:14:38,559 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/A of 6d8c50823ce967d5944d0259cb160d17 into b715ce85a4cc43b79845a79cae95cfe7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:38,559 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:38,559 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/A, priority=12, startTime=1731827678524; duration=0sec 2024-11-17T07:14:38,559 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:38,559 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:A 2024-11-17T07:14:38,559 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:38,560 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:38,560 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/C is initiating minor compaction (all files) 2024-11-17T07:14:38,560 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/C in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:38,560 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/14addfc3568d4a1691424c71c49a6019, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5da112d639a841f8aff096330cb3acd6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d764b5aa4d8048baa37cdb0b1e3cc582, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/ca1a58b9920d4a62865cd48f4d50635a] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=46.9 K 2024-11-17T07:14:38,561 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14addfc3568d4a1691424c71c49a6019, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731827675489 2024-11-17T07:14:38,561 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5da112d639a841f8aff096330cb3acd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731827675612 2024-11-17T07:14:38,562 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting d764b5aa4d8048baa37cdb0b1e3cc582, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731827676262 2024-11-17T07:14:38,562 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca1a58b9920d4a62865cd48f4d50635a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731827676943 2024-11-17T07:14:38,574 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#C#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:38,575 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/d1b3725bb25f4ee8ac30575b19f92927 is 50, key is test_row_0/C:col10/1731827676943/Put/seqid=0 2024-11-17T07:14:38,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742096_1272 (size=12139) 2024-11-17T07:14:38,585 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/d1b3725bb25f4ee8ac30575b19f92927 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d1b3725bb25f4ee8ac30575b19f92927 2024-11-17T07:14:38,589 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/C of 6d8c50823ce967d5944d0259cb160d17 into d1b3725bb25f4ee8ac30575b19f92927(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:38,590 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:38,590 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/C, priority=12, startTime=1731827678524; duration=0sec 2024-11-17T07:14:38,590 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:38,590 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:C 2024-11-17T07:14:38,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:38,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:14:38,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:38,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:38,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:38,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:38,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:38,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:38,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/5f910ea163da445e8f53c7adad440955 is 50, key is test_row_0/A:col10/1731827678729/Put/seqid=0 2024-11-17T07:14:38,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742097_1273 (size=14341) 2024-11-17T07:14:38,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827738759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827738763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,769 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827738764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-17T07:14:38,778 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:38,778 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-17T07:14:38,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T07:14:38,779 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:38,780 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:38,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:38,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827738867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827738877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827738877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:38,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T07:14:38,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T07:14:38,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:38,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:38,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:38,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:38,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:38,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:38,956 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/326a859ee39d4dee80c11417199ade3d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/326a859ee39d4dee80c11417199ade3d 2024-11-17T07:14:38,961 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/B of 6d8c50823ce967d5944d0259cb160d17 into 326a859ee39d4dee80c11417199ade3d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:38,961 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:38,961 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/B, priority=12, startTime=1731827678524; duration=0sec 2024-11-17T07:14:38,961 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:38,961 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:B 2024-11-17T07:14:39,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827739077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:39,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:39,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T07:14:39,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:39,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:39,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:39,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827739088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:39,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827739088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T07:14:39,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827739107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:39,110 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in 6d8c50823ce967d5944d0259cb160d17 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:39,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827739117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:39,119 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in 6d8c50823ce967d5944d0259cb160d17 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:39,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/5f910ea163da445e8f53c7adad440955 2024-11-17T07:14:39,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/5229a4f1e10549b8bfbadda157ffe507 is 50, key is test_row_0/B:col10/1731827678729/Put/seqid=0 2024-11-17T07:14:39,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742098_1274 (size=12001) 2024-11-17T07:14:39,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T07:14:39,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:39,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:39,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:39,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:39,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827739387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:39,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T07:14:39,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:39,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:39,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:39,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827739397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:39,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827739397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T07:14:39,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T07:14:39,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:39,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/5229a4f1e10549b8bfbadda157ffe507 2024-11-17T07:14:39,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/e18ff74a638c49cabeeef00de606fee4 is 50, key is test_row_0/C:col10/1731827678729/Put/seqid=0 2024-11-17T07:14:39,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742099_1275 (size=12001) 2024-11-17T07:14:39,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T07:14:39,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:39,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:39,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:39,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T07:14:39,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:39,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:39,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:39,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:39,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827739897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:39,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827739907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:39,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827739907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:39,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T07:14:39,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/e18ff74a638c49cabeeef00de606fee4 2024-11-17T07:14:39,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/5f910ea163da445e8f53c7adad440955 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5f910ea163da445e8f53c7adad440955 2024-11-17T07:14:39,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5f910ea163da445e8f53c7adad440955, entries=200, sequenceid=111, filesize=14.0 K 2024-11-17T07:14:39,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/5229a4f1e10549b8bfbadda157ffe507 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/5229a4f1e10549b8bfbadda157ffe507 2024-11-17T07:14:39,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/5229a4f1e10549b8bfbadda157ffe507, entries=150, sequenceid=111, filesize=11.7 K 2024-11-17T07:14:39,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/e18ff74a638c49cabeeef00de606fee4 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e18ff74a638c49cabeeef00de606fee4 2024-11-17T07:14:39,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e18ff74a638c49cabeeef00de606fee4, entries=150, sequenceid=111, filesize=11.7 K 2024-11-17T07:14:39,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6d8c50823ce967d5944d0259cb160d17 in 1263ms, sequenceid=111, compaction requested=false 2024-11-17T07:14:39,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:40,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-17T07:14:40,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:40,007 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T07:14:40,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:40,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:40,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:40,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:40,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:40,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:40,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/8ae2778c4b7a499f9c6ce8733bc0b53a is 50, key is test_row_0/A:col10/1731827678757/Put/seqid=0 2024-11-17T07:14:40,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742100_1276 (size=12051) 2024-11-17T07:14:40,016 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/8ae2778c4b7a499f9c6ce8733bc0b53a 2024-11-17T07:14:40,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/6b8ee3f9fd77416fa3cd488902377b36 is 50, key is test_row_0/B:col10/1731827678757/Put/seqid=0 2024-11-17T07:14:40,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742101_1277 (size=12051) 2024-11-17T07:14:40,431 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/6b8ee3f9fd77416fa3cd488902377b36 2024-11-17T07:14:40,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/d486a7186bec4d1e9f0d6cc3bfbf9843 is 50, key is test_row_0/C:col10/1731827678757/Put/seqid=0 2024-11-17T07:14:40,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742102_1278 (size=12051) 2024-11-17T07:14:40,844 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/d486a7186bec4d1e9f0d6cc3bfbf9843 2024-11-17T07:14:40,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/8ae2778c4b7a499f9c6ce8733bc0b53a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/8ae2778c4b7a499f9c6ce8733bc0b53a 2024-11-17T07:14:40,855 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/8ae2778c4b7a499f9c6ce8733bc0b53a, entries=150, sequenceid=131, filesize=11.8 K 2024-11-17T07:14:40,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/6b8ee3f9fd77416fa3cd488902377b36 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/6b8ee3f9fd77416fa3cd488902377b36 2024-11-17T07:14:40,862 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/6b8ee3f9fd77416fa3cd488902377b36, entries=150, sequenceid=131, filesize=11.8 K 2024-11-17T07:14:40,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/d486a7186bec4d1e9f0d6cc3bfbf9843 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d486a7186bec4d1e9f0d6cc3bfbf9843 2024-11-17T07:14:40,868 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d486a7186bec4d1e9f0d6cc3bfbf9843, entries=150, sequenceid=131, filesize=11.8 K 2024-11-17T07:14:40,869 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 6d8c50823ce967d5944d0259cb160d17 in 862ms, sequenceid=131, compaction requested=true 2024-11-17T07:14:40,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:40,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:40,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-17T07:14:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-17T07:14:40,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-17T07:14:40,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0910 sec 2024-11-17T07:14:40,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.0950 sec 2024-11-17T07:14:40,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:40,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:14:40,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:40,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:40,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:40,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:40,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:40,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:40,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-17T07:14:40,928 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:40,928 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:40,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-17T07:14:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T07:14:40,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/6ab0a23dea90435e83fba330111f03ec is 50, key is test_row_0/A:col10/1731827680923/Put/seqid=0 2024-11-17T07:14:40,931 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:40,932 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:40,933 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:40,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742103_1279 (size=14541) 2024-11-17T07:14:40,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:40,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827740959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:40,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827680948, max timestamp=1731827680961Num uniques 859; 2024-11-17T07:14:40,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827680948, max timestamp=1731827680961Num uniques 859; 2024-11-17T07:14:40,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827680948, max timestamp=1731827680961Num uniques 859; 2024-11-17T07:14:40,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:14:40,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:14:40,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:14:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:40,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827741001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827741005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T07:14:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827741077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T07:14:41,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:41,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:41,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:41,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827741117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:41,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827741117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:41,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827741117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,121 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in 6d8c50823ce967d5944d0259cb160d17 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827741137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:41,137 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in 6d8c50823ce967d5944d0259cb160d17 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T07:14:41,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:41,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:41,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:41,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T07:14:41,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:41,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827741287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827741328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827741328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/6ab0a23dea90435e83fba330111f03ec 2024-11-17T07:14:41,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/999a080ff43146c8a7d4bbb49d673582 is 50, key is test_row_0/B:col10/1731827680923/Put/seqid=0 2024-11-17T07:14:41,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742104_1280 (size=12151) 2024-11-17T07:14:41,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T07:14:41,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:41,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:41,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:41,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T07:14:41,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:41,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:41,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:41,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T07:14:41,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:41,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827741597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827741637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827741637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:41,643 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:41,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T07:14:41,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:41,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:41,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:41,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/999a080ff43146c8a7d4bbb49d673582 2024-11-17T07:14:41,758 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/539c7d6eccde4a15a2b021a0cc7698d5 is 50, key is test_row_0/C:col10/1731827680923/Put/seqid=0 2024-11-17T07:14:41,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742105_1281 (size=12151) 2024-11-17T07:14:41,851 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T07:14:41,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:41,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:41,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:41,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:41,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:42,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T07:14:42,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:42,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:42,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:42,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:42,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:42,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:42,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T07:14:42,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:42,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827742107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:42,110 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:42,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827742147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:42,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:42,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T07:14:42,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:42,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:42,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:42,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:42,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:42,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827742157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:42,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:42,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:42,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:42,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/539c7d6eccde4a15a2b021a0cc7698d5 2024-11-17T07:14:42,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/6ab0a23dea90435e83fba330111f03ec as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/6ab0a23dea90435e83fba330111f03ec 2024-11-17T07:14:42,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/6ab0a23dea90435e83fba330111f03ec, entries=200, sequenceid=141, filesize=14.2 K 2024-11-17T07:14:42,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/999a080ff43146c8a7d4bbb49d673582 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/999a080ff43146c8a7d4bbb49d673582 2024-11-17T07:14:42,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/999a080ff43146c8a7d4bbb49d673582, entries=150, sequenceid=141, filesize=11.9 K 2024-11-17T07:14:42,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/539c7d6eccde4a15a2b021a0cc7698d5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/539c7d6eccde4a15a2b021a0cc7698d5 2024-11-17T07:14:42,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/539c7d6eccde4a15a2b021a0cc7698d5, entries=150, sequenceid=141, filesize=11.9 K 2024-11-17T07:14:42,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6d8c50823ce967d5944d0259cb160d17 in 1269ms, sequenceid=141, compaction requested=true 2024-11-17T07:14:42,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:42,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:42,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:42,196 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:42,196 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:42,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:42,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:42,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:42,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:42,197 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:42,197 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53072 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:42,197 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/B is initiating minor compaction (all files) 2024-11-17T07:14:42,197 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/A is initiating minor compaction (all files) 2024-11-17T07:14:42,197 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/A in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:42,197 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/B in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:42,197 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/b715ce85a4cc43b79845a79cae95cfe7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5f910ea163da445e8f53c7adad440955, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/8ae2778c4b7a499f9c6ce8733bc0b53a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/6ab0a23dea90435e83fba330111f03ec] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=51.8 K 2024-11-17T07:14:42,198 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/326a859ee39d4dee80c11417199ade3d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/5229a4f1e10549b8bfbadda157ffe507, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/6b8ee3f9fd77416fa3cd488902377b36, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/999a080ff43146c8a7d4bbb49d673582] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=47.2 K 2024-11-17T07:14:42,198 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting b715ce85a4cc43b79845a79cae95cfe7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731827676943 2024-11-17T07:14:42,198 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 326a859ee39d4dee80c11417199ade3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731827676943 2024-11-17T07:14:42,198 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f910ea163da445e8f53c7adad440955, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731827678091 2024-11-17T07:14:42,198 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 5229a4f1e10549b8bfbadda157ffe507, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731827678728 2024-11-17T07:14:42,199 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ae2778c4b7a499f9c6ce8733bc0b53a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731827678757 2024-11-17T07:14:42,199 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b8ee3f9fd77416fa3cd488902377b36, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731827678757 2024-11-17T07:14:42,199 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ab0a23dea90435e83fba330111f03ec, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731827680921 2024-11-17T07:14:42,199 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 999a080ff43146c8a7d4bbb49d673582, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731827680922 2024-11-17T07:14:42,208 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#A#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:42,209 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/a96e0343b9a646f5be257bed3a4c126f is 50, key is test_row_0/A:col10/1731827680923/Put/seqid=0 2024-11-17T07:14:42,213 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#B#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:42,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742106_1282 (size=12425) 2024-11-17T07:14:42,213 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/8f8fc5e90623443b867a7e8b51692717 is 50, key is test_row_0/B:col10/1731827680923/Put/seqid=0 2024-11-17T07:14:42,219 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/a96e0343b9a646f5be257bed3a4c126f as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/a96e0343b9a646f5be257bed3a4c126f 2024-11-17T07:14:42,226 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/A of 6d8c50823ce967d5944d0259cb160d17 into a96e0343b9a646f5be257bed3a4c126f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:42,226 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:42,226 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/A, priority=12, startTime=1731827682196; duration=0sec 2024-11-17T07:14:42,226 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:42,226 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:A 2024-11-17T07:14:42,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742107_1283 (size=12425) 2024-11-17T07:14:42,226 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:42,229 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:42,229 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/C is initiating minor compaction (all files) 2024-11-17T07:14:42,229 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/C in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:42,229 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d1b3725bb25f4ee8ac30575b19f92927, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e18ff74a638c49cabeeef00de606fee4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d486a7186bec4d1e9f0d6cc3bfbf9843, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/539c7d6eccde4a15a2b021a0cc7698d5] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=47.2 K 2024-11-17T07:14:42,230 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1b3725bb25f4ee8ac30575b19f92927, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731827676943 2024-11-17T07:14:42,231 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting e18ff74a638c49cabeeef00de606fee4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1731827678728 2024-11-17T07:14:42,231 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting d486a7186bec4d1e9f0d6cc3bfbf9843, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731827678757 2024-11-17T07:14:42,232 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 539c7d6eccde4a15a2b021a0cc7698d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731827680922 2024-11-17T07:14:42,234 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/8f8fc5e90623443b867a7e8b51692717 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/8f8fc5e90623443b867a7e8b51692717 2024-11-17T07:14:42,241 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/B of 6d8c50823ce967d5944d0259cb160d17 into 8f8fc5e90623443b867a7e8b51692717(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:42,241 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:42,241 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/B, priority=12, startTime=1731827682196; duration=0sec 2024-11-17T07:14:42,241 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:42,241 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:B 2024-11-17T07:14:42,247 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#C#compaction#236 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:42,247 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/5df402122ebf44e0b477eb8613f1b7f0 is 50, key is test_row_0/C:col10/1731827680923/Put/seqid=0 2024-11-17T07:14:42,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742108_1284 (size=12425) 2024-11-17T07:14:42,277 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/5df402122ebf44e0b477eb8613f1b7f0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5df402122ebf44e0b477eb8613f1b7f0 2024-11-17T07:14:42,282 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/C of 6d8c50823ce967d5944d0259cb160d17 into 5df402122ebf44e0b477eb8613f1b7f0(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:42,282 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:42,282 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/C, priority=12, startTime=1731827682196; duration=0sec 2024-11-17T07:14:42,282 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:42,282 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:C 2024-11-17T07:14:42,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-17T07:14:42,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:42,309 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-17T07:14:42,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:42,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:42,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:42,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:42,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:42,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:42,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/7e28f628ad0746508c3ce0f91cd93a60 is 50, key is test_row_0/A:col10/1731827680961/Put/seqid=0 2024-11-17T07:14:42,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742109_1285 (size=12151) 2024-11-17T07:14:42,730 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/7e28f628ad0746508c3ce0f91cd93a60 2024-11-17T07:14:42,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/84922ca228a14149bce6379206218ad3 is 50, key is test_row_0/B:col10/1731827680961/Put/seqid=0 2024-11-17T07:14:42,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742110_1286 (size=12151) 2024-11-17T07:14:42,743 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/84922ca228a14149bce6379206218ad3 2024-11-17T07:14:42,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/e176277b329f4ac48009c5f52dd0e2ce is 50, key is test_row_0/C:col10/1731827680961/Put/seqid=0 2024-11-17T07:14:42,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742111_1287 (size=12151) 2024-11-17T07:14:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T07:14:43,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:43,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:43,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:43,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827743150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:43,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:43,156 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/e176277b329f4ac48009c5f52dd0e2ce 2024-11-17T07:14:43,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/7e28f628ad0746508c3ce0f91cd93a60 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/7e28f628ad0746508c3ce0f91cd93a60 2024-11-17T07:14:43,165 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/7e28f628ad0746508c3ce0f91cd93a60, entries=150, sequenceid=179, filesize=11.9 K 2024-11-17T07:14:43,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/84922ca228a14149bce6379206218ad3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/84922ca228a14149bce6379206218ad3 2024-11-17T07:14:43,170 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/84922ca228a14149bce6379206218ad3, entries=150, sequenceid=179, filesize=11.9 K 2024-11-17T07:14:43,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:43,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827743167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:43,171 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in 6d8c50823ce967d5944d0259cb160d17 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:43,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:43,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/e176277b329f4ac48009c5f52dd0e2ce as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e176277b329f4ac48009c5f52dd0e2ce 2024-11-17T07:14:43,176 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e176277b329f4ac48009c5f52dd0e2ce, entries=150, sequenceid=179, filesize=11.9 K 2024-11-17T07:14:43,177 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 6d8c50823ce967d5944d0259cb160d17 in 868ms, sequenceid=179, compaction requested=false 2024-11-17T07:14:43,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:43,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:43,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-17T07:14:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-17T07:14:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:43,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:14:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:43,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:43,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-17T07:14:43,184 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2500 sec 2024-11-17T07:14:43,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/b505bfd7b5b041268c8095d7b46b58dd is 50, key is test_row_0/A:col10/1731827683142/Put/seqid=0 2024-11-17T07:14:43,187 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.2570 sec 2024-11-17T07:14:43,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742112_1288 (size=12151) 2024-11-17T07:14:43,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/b505bfd7b5b041268c8095d7b46b58dd 2024-11-17T07:14:43,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/90982646fefd47d2b63fec4658dda753 is 50, key is test_row_0/B:col10/1731827683142/Put/seqid=0 2024-11-17T07:14:43,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742113_1289 (size=12151) 2024-11-17T07:14:43,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:43,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827743258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:43,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827683186, max timestamp=1731827683260Num uniques 859; 2024-11-17T07:14:43,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827683186, max timestamp=1731827683260Num uniques 859; 2024-11-17T07:14:43,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827683186, max timestamp=1731827683260Num uniques 859; 2024-11-17T07:14:43,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:14:43,263 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:14:43,264 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:14:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:43,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827743336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:43,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:43,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827743367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,371 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:43,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:43,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827743447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:43,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827743577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:43,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/90982646fefd47d2b63fec4658dda753 2024-11-17T07:14:43,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/397c5dda280e4451a38a932faab1742f is 50, key is test_row_0/C:col10/1731827683142/Put/seqid=0 2024-11-17T07:14:43,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742114_1290 (size=12151) 2024-11-17T07:14:43,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827743657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:43,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827743887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:43,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:43,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:43,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827743967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:43,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:44,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/397c5dda280e4451a38a932faab1742f 2024-11-17T07:14:44,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/b505bfd7b5b041268c8095d7b46b58dd as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/b505bfd7b5b041268c8095d7b46b58dd 2024-11-17T07:14:44,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/b505bfd7b5b041268c8095d7b46b58dd, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T07:14:44,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/90982646fefd47d2b63fec4658dda753 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/90982646fefd47d2b63fec4658dda753 2024-11-17T07:14:44,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/90982646fefd47d2b63fec4658dda753, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T07:14:44,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/397c5dda280e4451a38a932faab1742f as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/397c5dda280e4451a38a932faab1742f 2024-11-17T07:14:44,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/397c5dda280e4451a38a932faab1742f, entries=150, sequenceid=190, filesize=11.9 K 2024-11-17T07:14:44,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6d8c50823ce967d5944d0259cb160d17 in 860ms, sequenceid=190, compaction requested=true 2024-11-17T07:14:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:44,041 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:44,041 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:44,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:44,042 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:44,042 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:44,042 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/A is initiating minor compaction (all files) 2024-11-17T07:14:44,042 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/B is initiating minor compaction (all files) 2024-11-17T07:14:44,042 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/B in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:44,042 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/A in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:44,042 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/a96e0343b9a646f5be257bed3a4c126f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/7e28f628ad0746508c3ce0f91cd93a60, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/b505bfd7b5b041268c8095d7b46b58dd] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=35.9 K 2024-11-17T07:14:44,042 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/8f8fc5e90623443b867a7e8b51692717, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/84922ca228a14149bce6379206218ad3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/90982646fefd47d2b63fec4658dda753] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=35.9 K 2024-11-17T07:14:44,043 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f8fc5e90623443b867a7e8b51692717, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731827680922 2024-11-17T07:14:44,043 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting a96e0343b9a646f5be257bed3a4c126f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731827680922 2024-11-17T07:14:44,043 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e28f628ad0746508c3ce0f91cd93a60, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731827680961 2024-11-17T07:14:44,043 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 84922ca228a14149bce6379206218ad3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731827680961 2024-11-17T07:14:44,044 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting b505bfd7b5b041268c8095d7b46b58dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731827683142 2024-11-17T07:14:44,045 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 90982646fefd47d2b63fec4658dda753, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731827683142 2024-11-17T07:14:44,063 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#B#compaction#244 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:44,063 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#A#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:44,063 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/a907b5a95c7f4d398fdd1c46a50c78e1 is 50, key is test_row_0/B:col10/1731827683142/Put/seqid=0 2024-11-17T07:14:44,063 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/4b49d8c5ce4447578b4a59ff58d7db2b is 50, key is test_row_0/A:col10/1731827683142/Put/seqid=0 2024-11-17T07:14:44,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742116_1292 (size=12527) 2024-11-17T07:14:44,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742115_1291 (size=12527) 2024-11-17T07:14:44,387 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T07:14:44,388 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T07:14:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:44,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-17T07:14:44,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:44,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:44,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:44,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:44,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:44,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:44,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/3e3da9bbca3d46358917858c4c3b84d0 is 50, key is test_row_0/A:col10/1731827683322/Put/seqid=0 2024-11-17T07:14:44,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742117_1293 (size=14541) 2024-11-17T07:14:44,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827744460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:44,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:44,478 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/4b49d8c5ce4447578b4a59ff58d7db2b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4b49d8c5ce4447578b4a59ff58d7db2b 2024-11-17T07:14:44,483 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/A of 6d8c50823ce967d5944d0259cb160d17 into 4b49d8c5ce4447578b4a59ff58d7db2b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:44,483 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:44,483 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/A, priority=13, startTime=1731827684041; duration=0sec 2024-11-17T07:14:44,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:44,483 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:44,483 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:A 2024-11-17T07:14:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827744477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:44,483 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/a907b5a95c7f4d398fdd1c46a50c78e1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a907b5a95c7f4d398fdd1c46a50c78e1 2024-11-17T07:14:44,483 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:44,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:44,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:44,485 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:44,485 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/C is initiating minor compaction (all files) 2024-11-17T07:14:44,485 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/C in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:44,485 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5df402122ebf44e0b477eb8613f1b7f0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e176277b329f4ac48009c5f52dd0e2ce, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/397c5dda280e4451a38a932faab1742f] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=35.9 K 2024-11-17T07:14:44,486 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5df402122ebf44e0b477eb8613f1b7f0, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731827680922 2024-11-17T07:14:44,486 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting e176277b329f4ac48009c5f52dd0e2ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731827680961 2024-11-17T07:14:44,486 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 397c5dda280e4451a38a932faab1742f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731827683142 2024-11-17T07:14:44,489 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/B of 6d8c50823ce967d5944d0259cb160d17 into a907b5a95c7f4d398fdd1c46a50c78e1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:44,489 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:44,489 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/B, priority=13, startTime=1731827684041; duration=0sec 2024-11-17T07:14:44,489 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:44,489 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:B 2024-11-17T07:14:44,493 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:44,493 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/9e2802f856634dcdb85b7afa323262c0 is 50, key is test_row_0/C:col10/1731827683142/Put/seqid=0 2024-11-17T07:14:44,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742118_1294 (size=12527) 2024-11-17T07:14:44,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827744567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:44,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:44,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:44,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:44,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827744778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:44,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/3e3da9bbca3d46358917858c4c3b84d0 2024-11-17T07:14:44,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/77c08d044cfd4a79935719c7815c06ed is 50, key is test_row_0/B:col10/1731827683322/Put/seqid=0 2024-11-17T07:14:44,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742119_1295 (size=12151) 2024-11-17T07:14:44,919 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/9e2802f856634dcdb85b7afa323262c0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/9e2802f856634dcdb85b7afa323262c0 2024-11-17T07:14:44,924 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/C of 6d8c50823ce967d5944d0259cb160d17 into 9e2802f856634dcdb85b7afa323262c0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:44,924 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:44,924 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/C, priority=13, startTime=1731827684041; duration=0sec 2024-11-17T07:14:44,924 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:44,924 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:C 2024-11-17T07:14:45,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827745087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-17T07:14:45,108 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:45,108 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:45,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-17T07:14:45,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T07:14:45,111 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:45,111 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:45,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:45,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:45,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827745147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:45,149 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in 6d8c50823ce967d5944d0259cb160d17 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:45,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:45,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:45,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827745167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:45,170 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in 6d8c50823ce967d5944d0259cb160d17 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:45,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:45,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827745197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:45,201 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in 6d8c50823ce967d5944d0259cb160d17 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:45,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T07:14:45,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/77c08d044cfd4a79935719c7815c06ed 2024-11-17T07:14:45,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/2e12dbf5e7364d7e825361288db63bf1 is 50, key is test_row_0/C:col10/1731827683322/Put/seqid=0 2024-11-17T07:14:45,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742120_1296 (size=12151) 2024-11-17T07:14:45,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T07:14:45,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:45,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:45,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:45,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:45,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:45,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T07:14:45,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:45,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:45,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:45,418 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:45,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:45,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T07:14:45,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:45,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827745497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:45,501 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in 6d8c50823ce967d5944d0259cb160d17 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:45,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:45,571 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T07:14:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:45,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:45,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:45,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:45,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:45,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827745597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:45,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:45,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/2e12dbf5e7364d7e825361288db63bf1 2024-11-17T07:14:45,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/3e3da9bbca3d46358917858c4c3b84d0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3e3da9bbca3d46358917858c4c3b84d0 2024-11-17T07:14:45,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3e3da9bbca3d46358917858c4c3b84d0, entries=200, sequenceid=226, filesize=14.2 K 2024-11-17T07:14:45,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/77c08d044cfd4a79935719c7815c06ed as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/77c08d044cfd4a79935719c7815c06ed 2024-11-17T07:14:45,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/77c08d044cfd4a79935719c7815c06ed, entries=150, sequenceid=226, filesize=11.9 K 2024-11-17T07:14:45,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/2e12dbf5e7364d7e825361288db63bf1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/2e12dbf5e7364d7e825361288db63bf1 2024-11-17T07:14:45,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/2e12dbf5e7364d7e825361288db63bf1, entries=150, sequenceid=226, filesize=11.9 K 2024-11-17T07:14:45,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6d8c50823ce967d5944d0259cb160d17 in 1275ms, sequenceid=226, compaction requested=false 2024-11-17T07:14:45,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:45,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-17T07:14:45,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:45,726 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:14:45,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:45,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:45,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:45,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:45,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:45,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:45,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/8c36010c586c43d9a5da7630db0188aa is 50, key is test_row_0/A:col10/1731827684445/Put/seqid=0 2024-11-17T07:14:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T07:14:45,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742121_1297 (size=12151) 2024-11-17T07:14:45,768 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/8c36010c586c43d9a5da7630db0188aa 2024-11-17T07:14:45,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/46a247653de94a9793b83b246f877444 is 50, key is test_row_0/B:col10/1731827684445/Put/seqid=0 2024-11-17T07:14:45,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742122_1298 (size=12151) 2024-11-17T07:14:45,828 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/46a247653de94a9793b83b246f877444 2024-11-17T07:14:45,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/6705e0b68ef143afa204fb22b4d8c2d0 is 50, key is test_row_0/C:col10/1731827684445/Put/seqid=0 2024-11-17T07:14:45,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742123_1299 (size=12151) 2024-11-17T07:14:45,886 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/6705e0b68ef143afa204fb22b4d8c2d0 2024-11-17T07:14:45,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/8c36010c586c43d9a5da7630db0188aa as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/8c36010c586c43d9a5da7630db0188aa 2024-11-17T07:14:45,903 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/8c36010c586c43d9a5da7630db0188aa, entries=150, sequenceid=239, filesize=11.9 K 2024-11-17T07:14:45,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/46a247653de94a9793b83b246f877444 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/46a247653de94a9793b83b246f877444 2024-11-17T07:14:45,908 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/46a247653de94a9793b83b246f877444, entries=150, sequenceid=239, filesize=11.9 K 2024-11-17T07:14:45,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/6705e0b68ef143afa204fb22b4d8c2d0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/6705e0b68ef143afa204fb22b4d8c2d0 2024-11-17T07:14:45,914 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/6705e0b68ef143afa204fb22b4d8c2d0, entries=150, sequenceid=239, filesize=11.9 K 2024-11-17T07:14:45,915 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 6d8c50823ce967d5944d0259cb160d17 in 190ms, sequenceid=239, compaction requested=true 2024-11-17T07:14:45,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:45,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:45,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-17T07:14:45,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-17T07:14:45,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-17T07:14:45,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 805 msec 2024-11-17T07:14:45,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 812 msec 2024-11-17T07:14:46,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-17T07:14:46,247 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:46,248 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-17T07:14:46,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T07:14:46,250 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:46,250 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:46,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T07:14:46,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-17T07:14:46,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:46,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:46,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:46,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-17T07:14:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-17T07:14:46,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-17T07:14:46,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 155 msec 2024-11-17T07:14:46,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 160 msec 2024-11-17T07:14:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-17T07:14:46,567 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:46,567 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-17T07:14:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T07:14:46,570 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:46,570 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:46,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:46,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:46,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:14:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:46,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:46,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/3864c2cf1b9a4500906dfc79ae56f595 is 50, key is test_row_0/A:col10/1731827686621/Put/seqid=0 2024-11-17T07:14:46,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742124_1300 (size=14541) 2024-11-17T07:14:46,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T07:14:46,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827686689, max timestamp=1731827686714Num uniques 859; 2024-11-17T07:14:46,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:46,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827686689, max timestamp=1731827686714Num uniques 859; 2024-11-17T07:14:46,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:46,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827686689, max timestamp=1731827686714Num uniques 859; 2024-11-17T07:14:46,718 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:46,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T07:14:46,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:46,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:46,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:46,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:46,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:46,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:46,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:46,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827746813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:46,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:46,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T07:14:46,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:46,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:46,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:46,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:46,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:46,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:46,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T07:14:46,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:46,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827746927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:46,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:46,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:47,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T07:14:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/3864c2cf1b9a4500906dfc79ae56f595 2024-11-17T07:14:47,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/de3fda9427254430a120ed4fcbd6b8de is 50, key is test_row_0/B:col10/1731827686621/Put/seqid=0 2024-11-17T07:14:47,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742125_1301 (size=12151) 2024-11-17T07:14:47,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827747137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:47,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:47,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:47,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:47,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T07:14:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:47,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T07:14:47,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T07:14:47,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:47,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:47,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827747448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:47,453 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:47,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/de3fda9427254430a120ed4fcbd6b8de 2024-11-17T07:14:47,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/06672b322a424068adf3dbe163d56209 is 50, key is test_row_0/C:col10/1731827686621/Put/seqid=0 2024-11-17T07:14:47,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742126_1302 (size=12151) 2024-11-17T07:14:47,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T07:14:47,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:47,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:47,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827747517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:47,522 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in 6d8c50823ce967d5944d0259cb160d17 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:47,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:47,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:47,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:47,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T07:14:47,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:47,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T07:14:47,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T07:14:47,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:47,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:47,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/06672b322a424068adf3dbe163d56209 2024-11-17T07:14:47,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/3864c2cf1b9a4500906dfc79ae56f595 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3864c2cf1b9a4500906dfc79ae56f595 2024-11-17T07:14:47,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3864c2cf1b9a4500906dfc79ae56f595, entries=200, sequenceid=250, filesize=14.2 K 2024-11-17T07:14:47,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/de3fda9427254430a120ed4fcbd6b8de as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/de3fda9427254430a120ed4fcbd6b8de 2024-11-17T07:14:47,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/de3fda9427254430a120ed4fcbd6b8de, entries=150, sequenceid=250, filesize=11.9 K 2024-11-17T07:14:47,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/06672b322a424068adf3dbe163d56209 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/06672b322a424068adf3dbe163d56209 2024-11-17T07:14:47,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/06672b322a424068adf3dbe163d56209, entries=150, sequenceid=250, filesize=11.9 K 2024-11-17T07:14:47,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6d8c50823ce967d5944d0259cb160d17 in 1270ms, sequenceid=250, compaction requested=true 2024-11-17T07:14:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:47,903 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:47,903 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:47,903 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:47,904 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53760 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:47,904 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/A is initiating minor compaction (all files) 2024-11-17T07:14:47,904 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/A in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,905 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4b49d8c5ce4447578b4a59ff58d7db2b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3e3da9bbca3d46358917858c4c3b84d0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/8c36010c586c43d9a5da7630db0188aa, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3864c2cf1b9a4500906dfc79ae56f595] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=52.5 K 2024-11-17T07:14:47,905 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:47,905 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b49d8c5ce4447578b4a59ff58d7db2b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731827683142 2024-11-17T07:14:47,905 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/B is initiating minor compaction (all files) 2024-11-17T07:14:47,905 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/B in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,905 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a907b5a95c7f4d398fdd1c46a50c78e1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/77c08d044cfd4a79935719c7815c06ed, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/46a247653de94a9793b83b246f877444, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/de3fda9427254430a120ed4fcbd6b8de] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=47.8 K 2024-11-17T07:14:47,905 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e3da9bbca3d46358917858c4c3b84d0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731827683322 2024-11-17T07:14:47,906 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c36010c586c43d9a5da7630db0188aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731827684445 2024-11-17T07:14:47,906 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a907b5a95c7f4d398fdd1c46a50c78e1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731827683142 2024-11-17T07:14:47,906 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3864c2cf1b9a4500906dfc79ae56f595, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731827686621 2024-11-17T07:14:47,906 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 77c08d044cfd4a79935719c7815c06ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731827683322 2024-11-17T07:14:47,906 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 46a247653de94a9793b83b246f877444, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731827684445 2024-11-17T07:14:47,907 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting de3fda9427254430a120ed4fcbd6b8de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731827686621 2024-11-17T07:14:47,918 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#A#compaction#255 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:47,918 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#B#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:47,919 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/05ee820ad54d40138c0aa9a85e1c49d6 is 50, key is test_row_0/A:col10/1731827686621/Put/seqid=0 2024-11-17T07:14:47,919 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/33d5944538b94c16ba757cfd0a7841bb is 50, key is test_row_0/B:col10/1731827686621/Put/seqid=0 2024-11-17T07:14:47,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742127_1303 (size=12663) 2024-11-17T07:14:47,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742128_1304 (size=12663) 2024-11-17T07:14:47,945 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/33d5944538b94c16ba757cfd0a7841bb as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/33d5944538b94c16ba757cfd0a7841bb 2024-11-17T07:14:47,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-17T07:14:47,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,948 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-17T07:14:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:47,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:47,952 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/B of 6d8c50823ce967d5944d0259cb160d17 into 33d5944538b94c16ba757cfd0a7841bb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:47,952 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:47,952 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/B, priority=12, startTime=1731827687903; duration=0sec 2024-11-17T07:14:47,952 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:47,952 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:B 2024-11-17T07:14:47,952 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:14:47,953 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:14:47,953 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/C is initiating minor compaction (all files) 2024-11-17T07:14:47,953 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/C in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:47,954 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/9e2802f856634dcdb85b7afa323262c0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/2e12dbf5e7364d7e825361288db63bf1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/6705e0b68ef143afa204fb22b4d8c2d0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/06672b322a424068adf3dbe163d56209] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=47.8 K 2024-11-17T07:14:47,954 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e2802f856634dcdb85b7afa323262c0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1731827683142 2024-11-17T07:14:47,955 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e12dbf5e7364d7e825361288db63bf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731827683322 2024-11-17T07:14:47,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/ed6117189e2e467c821a0eff249c0f75 is 50, key is test_row_0/A:col10/1731827686811/Put/seqid=0 2024-11-17T07:14:47,955 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 6705e0b68ef143afa204fb22b4d8c2d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731827684445 2024-11-17T07:14:47,955 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 06672b322a424068adf3dbe163d56209, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731827686621 2024-11-17T07:14:47,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742129_1305 (size=12301) 2024-11-17T07:14:47,960 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/ed6117189e2e467c821a0eff249c0f75 2024-11-17T07:14:47,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:47,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:47,967 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#C#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:47,967 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/167bc983ca1844b397d3eaa0da2c2a23 is 50, key is test_row_0/C:col10/1731827686621/Put/seqid=0 2024-11-17T07:14:47,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/f088c1eee479457b8e0c7660454585c0 is 50, key is test_row_0/B:col10/1731827686811/Put/seqid=0 2024-11-17T07:14:47,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742130_1306 (size=12663) 2024-11-17T07:14:47,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742131_1307 (size=12301) 2024-11-17T07:14:48,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:48,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827748008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:48,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:48,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827748117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:48,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:48,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:48,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:48,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:48,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827748327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:48,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:48,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:48,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:48,344 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/05ee820ad54d40138c0aa9a85e1c49d6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/05ee820ad54d40138c0aa9a85e1c49d6 2024-11-17T07:14:48,349 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/A of 6d8c50823ce967d5944d0259cb160d17 into 05ee820ad54d40138c0aa9a85e1c49d6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:48,349 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:48,349 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/A, priority=12, startTime=1731827687903; duration=0sec 2024-11-17T07:14:48,350 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:48,350 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:A 2024-11-17T07:14:48,383 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/167bc983ca1844b397d3eaa0da2c2a23 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/167bc983ca1844b397d3eaa0da2c2a23 2024-11-17T07:14:48,387 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/C of 6d8c50823ce967d5944d0259cb160d17 into 167bc983ca1844b397d3eaa0da2c2a23(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:48,387 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:48,387 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/C, priority=12, startTime=1731827687903; duration=0sec 2024-11-17T07:14:48,388 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/f088c1eee479457b8e0c7660454585c0 2024-11-17T07:14:48,388 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:48,388 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:C 2024-11-17T07:14:48,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/781192e106f3466e9e63aa95b4758a7b is 50, key is test_row_0/C:col10/1731827686811/Put/seqid=0 2024-11-17T07:14:48,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742132_1308 (size=12301) 2024-11-17T07:14:48,399 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/781192e106f3466e9e63aa95b4758a7b 2024-11-17T07:14:48,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/ed6117189e2e467c821a0eff249c0f75 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/ed6117189e2e467c821a0eff249c0f75 2024-11-17T07:14:48,407 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/ed6117189e2e467c821a0eff249c0f75, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T07:14:48,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/f088c1eee479457b8e0c7660454585c0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/f088c1eee479457b8e0c7660454585c0 2024-11-17T07:14:48,412 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/f088c1eee479457b8e0c7660454585c0, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T07:14:48,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/781192e106f3466e9e63aa95b4758a7b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/781192e106f3466e9e63aa95b4758a7b 2024-11-17T07:14:48,416 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/781192e106f3466e9e63aa95b4758a7b, entries=150, sequenceid=287, filesize=12.0 K 2024-11-17T07:14:48,417 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 6d8c50823ce967d5944d0259cb160d17 in 469ms, sequenceid=287, compaction requested=false 2024-11-17T07:14:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:48,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-17T07:14:48,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-17T07:14:48,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-17T07:14:48,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8490 sec 2024-11-17T07:14:48,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.8540 sec 2024-11-17T07:14:48,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:48,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:14:48,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:48,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:48,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:48,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:48,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:48,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:48,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/9363dffb45f54f00afae025fc94f184c is 50, key is test_row_0/A:col10/1731827687990/Put/seqid=0 2024-11-17T07:14:48,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742133_1309 (size=12301) 2024-11-17T07:14:48,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/9363dffb45f54f00afae025fc94f184c 2024-11-17T07:14:48,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/ff193940c4a5481686a5bd718b33fe90 is 50, key is test_row_0/B:col10/1731827687990/Put/seqid=0 2024-11-17T07:14:48,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742134_1310 (size=12301) 2024-11-17T07:14:48,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-17T07:14:48,717 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:48,717 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:48,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-17T07:14:48,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T07:14:48,719 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:48,719 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:48,719 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:48,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827688707, max timestamp=1731827688736Num uniques 859; 2024-11-17T07:14:48,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827688707, max timestamp=1731827688736Num uniques 859; 2024-11-17T07:14:48,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:48,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:48,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827688707, max timestamp=1731827688736Num uniques 859; 2024-11-17T07:14:48,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:48,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827748817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:48,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T07:14:48,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T07:14:48,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:48,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:48,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:48,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:48,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:48,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:48,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:48,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827748927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:49,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T07:14:49,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:49,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:49,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:49,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:49,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:49,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T07:14:49,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/ff193940c4a5481686a5bd718b33fe90 2024-11-17T07:14:49,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/cbfecdaa1b364d12845b56ad03a42bf1 is 50, key is test_row_0/C:col10/1731827687990/Put/seqid=0 2024-11-17T07:14:49,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742135_1311 (size=12301) 2024-11-17T07:14:49,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:49,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827749137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:49,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T07:14:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:49,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:49,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:49,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:49,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:49,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41640 deadline: 1731827749237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:49,240 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in 6d8c50823ce967d5944d0259cb160d17 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:49,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:49,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T07:14:49,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:49,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:49,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:49,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:49,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:49,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T07:14:49,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827749447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:49,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T07:14:49,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:49,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:49,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:49,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:49,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:49,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/cbfecdaa1b364d12845b56ad03a42bf1 2024-11-17T07:14:49,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/9363dffb45f54f00afae025fc94f184c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/9363dffb45f54f00afae025fc94f184c 2024-11-17T07:14:49,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/9363dffb45f54f00afae025fc94f184c, entries=150, sequenceid=301, filesize=12.0 K 2024-11-17T07:14:49,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/ff193940c4a5481686a5bd718b33fe90 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/ff193940c4a5481686a5bd718b33fe90 2024-11-17T07:14:49,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/ff193940c4a5481686a5bd718b33fe90, entries=150, sequenceid=301, filesize=12.0 K 2024-11-17T07:14:49,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/cbfecdaa1b364d12845b56ad03a42bf1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/cbfecdaa1b364d12845b56ad03a42bf1 2024-11-17T07:14:49,503 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/cbfecdaa1b364d12845b56ad03a42bf1, entries=150, sequenceid=301, filesize=12.0 K 2024-11-17T07:14:49,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6d8c50823ce967d5944d0259cb160d17 in 853ms, sequenceid=301, compaction requested=true 2024-11-17T07:14:49,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:49,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:49,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:49,504 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:49,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:49,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:49,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:49,504 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:49,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:49,505 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:49,505 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:49,505 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/A is initiating minor compaction (all files) 2024-11-17T07:14:49,505 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/B is initiating minor compaction (all files) 2024-11-17T07:14:49,505 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/A in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:49,505 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/B in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:49,505 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/05ee820ad54d40138c0aa9a85e1c49d6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/ed6117189e2e467c821a0eff249c0f75, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/9363dffb45f54f00afae025fc94f184c] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=36.4 K 2024-11-17T07:14:49,505 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/33d5944538b94c16ba757cfd0a7841bb, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/f088c1eee479457b8e0c7660454585c0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/ff193940c4a5481686a5bd718b33fe90] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=36.4 K 2024-11-17T07:14:49,506 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05ee820ad54d40138c0aa9a85e1c49d6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731827686621 2024-11-17T07:14:49,506 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d5944538b94c16ba757cfd0a7841bb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731827686621 2024-11-17T07:14:49,506 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed6117189e2e467c821a0eff249c0f75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731827686792 2024-11-17T07:14:49,506 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting f088c1eee479457b8e0c7660454585c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731827686792 2024-11-17T07:14:49,506 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9363dffb45f54f00afae025fc94f184c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731827687973 2024-11-17T07:14:49,506 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting ff193940c4a5481686a5bd718b33fe90, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731827687973 2024-11-17T07:14:49,513 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:49,513 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#A#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:49,513 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/bd1abd38a59349fb91df7fcecff9072a is 50, key is test_row_0/B:col10/1731827687990/Put/seqid=0 2024-11-17T07:14:49,513 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/632f5246ff544083a108367daaabc08e is 50, key is test_row_0/A:col10/1731827687990/Put/seqid=0 2024-11-17T07:14:49,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742136_1312 (size=12915) 2024-11-17T07:14:49,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742137_1313 (size=12915) 2024-11-17T07:14:49,638 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-17T07:14:49,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:49,638 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-17T07:14:49,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:49,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:49,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/debf283689674ba89ee9bb3c2e72c4e3 is 50, key is test_row_0/A:col10/1731827688797/Put/seqid=0 2024-11-17T07:14:49,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742138_1314 (size=12301) 2024-11-17T07:14:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T07:14:49,923 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/bd1abd38a59349fb91df7fcecff9072a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/bd1abd38a59349fb91df7fcecff9072a 2024-11-17T07:14:49,923 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/632f5246ff544083a108367daaabc08e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/632f5246ff544083a108367daaabc08e 2024-11-17T07:14:49,928 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/A of 6d8c50823ce967d5944d0259cb160d17 into 632f5246ff544083a108367daaabc08e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:49,928 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/B of 6d8c50823ce967d5944d0259cb160d17 into bd1abd38a59349fb91df7fcecff9072a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:49,928 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:49,928 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:49,928 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/A, priority=13, startTime=1731827689504; duration=0sec 2024-11-17T07:14:49,928 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/B, priority=13, startTime=1731827689504; duration=0sec 2024-11-17T07:14:49,928 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:49,928 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:A 2024-11-17T07:14:49,928 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:49,928 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:B 2024-11-17T07:14:49,928 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:49,929 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:49,929 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/C is initiating minor compaction (all files) 2024-11-17T07:14:49,930 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/C in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:49,930 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/167bc983ca1844b397d3eaa0da2c2a23, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/781192e106f3466e9e63aa95b4758a7b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/cbfecdaa1b364d12845b56ad03a42bf1] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=36.4 K 2024-11-17T07:14:49,930 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 167bc983ca1844b397d3eaa0da2c2a23, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731827686621 2024-11-17T07:14:49,930 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 781192e106f3466e9e63aa95b4758a7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731827686792 2024-11-17T07:14:49,931 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbfecdaa1b364d12845b56ad03a42bf1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731827687973 2024-11-17T07:14:49,939 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#C#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:49,940 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/afaa8cf3f3784a8bbbb3c328c7e62981 is 50, key is test_row_0/C:col10/1731827687990/Put/seqid=0 2024-11-17T07:14:49,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742139_1315 (size=12915) 2024-11-17T07:14:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:49,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:50,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827749996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:50,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:50,048 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/debf283689674ba89ee9bb3c2e72c4e3 2024-11-17T07:14:50,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/5fba3ee25bc44ace92259bd9de57e2a0 is 50, key is test_row_0/B:col10/1731827688797/Put/seqid=0 2024-11-17T07:14:50,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742140_1316 (size=12301) 2024-11-17T07:14:50,060 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/5fba3ee25bc44ace92259bd9de57e2a0 2024-11-17T07:14:50,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/01fedcaa494f4c40bcc9423f9f0f9f5c is 50, key is test_row_0/C:col10/1731827688797/Put/seqid=0 2024-11-17T07:14:50,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742141_1317 (size=12301) 2024-11-17T07:14:50,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:50,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827750107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:50,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:50,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:50,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827750317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:50,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:50,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:50,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:50,357 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/afaa8cf3f3784a8bbbb3c328c7e62981 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/afaa8cf3f3784a8bbbb3c328c7e62981 2024-11-17T07:14:50,362 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/C of 6d8c50823ce967d5944d0259cb160d17 into afaa8cf3f3784a8bbbb3c328c7e62981(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:50,362 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:50,362 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/C, priority=13, startTime=1731827689504; duration=0sec 2024-11-17T07:14:50,362 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:50,362 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:C 2024-11-17T07:14:50,479 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/01fedcaa494f4c40bcc9423f9f0f9f5c 2024-11-17T07:14:50,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/debf283689674ba89ee9bb3c2e72c4e3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/debf283689674ba89ee9bb3c2e72c4e3 2024-11-17T07:14:50,487 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/debf283689674ba89ee9bb3c2e72c4e3, entries=150, sequenceid=337, filesize=12.0 K 2024-11-17T07:14:50,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/5fba3ee25bc44ace92259bd9de57e2a0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/5fba3ee25bc44ace92259bd9de57e2a0 2024-11-17T07:14:50,492 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/5fba3ee25bc44ace92259bd9de57e2a0, entries=150, sequenceid=337, filesize=12.0 K 2024-11-17T07:14:50,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/01fedcaa494f4c40bcc9423f9f0f9f5c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/01fedcaa494f4c40bcc9423f9f0f9f5c 2024-11-17T07:14:50,496 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/01fedcaa494f4c40bcc9423f9f0f9f5c, entries=150, sequenceid=337, filesize=12.0 K 2024-11-17T07:14:50,497 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 6d8c50823ce967d5944d0259cb160d17 in 859ms, sequenceid=337, compaction requested=false 2024-11-17T07:14:50,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:50,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:50,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-17T07:14:50,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-17T07:14:50,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-17T07:14:50,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7820 sec 2024-11-17T07:14:50,512 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.7940 sec 2024-11-17T07:14:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:50,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T07:14:50,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:50,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:50,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:50,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:50,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:50,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:50,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/1e4ea9497a324c0c95abae7211ad433a is 50, key is test_row_0/A:col10/1731827690627/Put/seqid=0 2024-11-17T07:14:50,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742142_1318 (size=12301) 2024-11-17T07:14:50,641 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/1e4ea9497a324c0c95abae7211ad433a 2024-11-17T07:14:50,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/7dce06921b2248a9bc8ce0710d95549d is 50, key is test_row_0/B:col10/1731827690627/Put/seqid=0 2024-11-17T07:14:50,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742143_1319 (size=12301) 2024-11-17T07:14:50,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827750714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:50,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:50,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:50,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827750827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:50,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-17T07:14:50,877 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:50,878 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-17T07:14:50,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T07:14:50,880 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:50,881 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:50,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:50,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T07:14:51,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T07:14:51,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:51,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,033 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827751037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:51,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/7dce06921b2248a9bc8ce0710d95549d 2024-11-17T07:14:51,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/56e4c249006b460ba31357155a8beb72 is 50, key is test_row_0/C:col10/1731827690627/Put/seqid=0 2024-11-17T07:14:51,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742144_1320 (size=12301) 2024-11-17T07:14:51,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T07:14:51,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:51,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T07:14:51,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T07:14:51,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:51,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827751347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:51,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:51,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/56e4c249006b460ba31357155a8beb72 2024-11-17T07:14:51,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/1e4ea9497a324c0c95abae7211ad433a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1e4ea9497a324c0c95abae7211ad433a 2024-11-17T07:14:51,489 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1e4ea9497a324c0c95abae7211ad433a, entries=150, sequenceid=352, filesize=12.0 K 2024-11-17T07:14:51,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/7dce06921b2248a9bc8ce0710d95549d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/7dce06921b2248a9bc8ce0710d95549d 2024-11-17T07:14:51,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T07:14:51,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:51,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/7dce06921b2248a9bc8ce0710d95549d, entries=150, sequenceid=352, filesize=12.0 K 2024-11-17T07:14:51,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/56e4c249006b460ba31357155a8beb72 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/56e4c249006b460ba31357155a8beb72 2024-11-17T07:14:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/56e4c249006b460ba31357155a8beb72, entries=150, sequenceid=352, filesize=12.0 K 2024-11-17T07:14:51,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6d8c50823ce967d5944d0259cb160d17 in 870ms, sequenceid=352, compaction requested=true 2024-11-17T07:14:51,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:51,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:51,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:51,499 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:51,499 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:51,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:51,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:51,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:51,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:51,501 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:51,501 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/A is initiating minor compaction (all files) 2024-11-17T07:14:51,501 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/A in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,501 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/632f5246ff544083a108367daaabc08e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/debf283689674ba89ee9bb3c2e72c4e3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1e4ea9497a324c0c95abae7211ad433a] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=36.6 K 2024-11-17T07:14:51,501 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:51,501 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/B is initiating minor compaction (all files) 2024-11-17T07:14:51,501 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/B in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,501 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/bd1abd38a59349fb91df7fcecff9072a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/5fba3ee25bc44ace92259bd9de57e2a0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/7dce06921b2248a9bc8ce0710d95549d] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=36.6 K 2024-11-17T07:14:51,502 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 632f5246ff544083a108367daaabc08e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731827687973 2024-11-17T07:14:51,502 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting bd1abd38a59349fb91df7fcecff9072a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731827687973 2024-11-17T07:14:51,502 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting debf283689674ba89ee9bb3c2e72c4e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731827688774 2024-11-17T07:14:51,502 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fba3ee25bc44ace92259bd9de57e2a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731827688774 2024-11-17T07:14:51,502 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e4ea9497a324c0c95abae7211ad433a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731827689974 2024-11-17T07:14:51,503 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dce06921b2248a9bc8ce0710d95549d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731827689974 2024-11-17T07:14:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T07:14:51,513 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#A#compaction#273 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:51,514 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/3dfa84c035f84bc9b644abd27d256422 is 50, key is test_row_0/A:col10/1731827690627/Put/seqid=0 2024-11-17T07:14:51,523 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#B#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:51,523 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/6794ec1beeb94459902da590a414e7ea is 50, key is test_row_0/B:col10/1731827690627/Put/seqid=0 2024-11-17T07:14:51,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827690645, max timestamp=1731827691549Num uniques 859; 2024-11-17T07:14:51,552 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:14:51,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742145_1321 (size=13017) 2024-11-17T07:14:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:51,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-17T07:14:51,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:51,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:51,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:51,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:51,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:51,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:14:51,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742146_1322 (size=13017) 2024-11-17T07:14:51,576 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/6794ec1beeb94459902da590a414e7ea as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/6794ec1beeb94459902da590a414e7ea 2024-11-17T07:14:51,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/4070913c3c8f44a79c568c22876cc9da is 50, key is test_row_0/A:col10/1731827691549/Put/seqid=0 2024-11-17T07:14:51,584 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/B of 6d8c50823ce967d5944d0259cb160d17 into 6794ec1beeb94459902da590a414e7ea(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:51,584 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:51,584 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/B, priority=13, startTime=1731827691499; duration=0sec 2024-11-17T07:14:51,585 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:51,585 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:B 2024-11-17T07:14:51,585 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:51,586 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:51,586 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/C is initiating minor compaction (all files) 2024-11-17T07:14:51,586 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/C in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,586 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/afaa8cf3f3784a8bbbb3c328c7e62981, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/01fedcaa494f4c40bcc9423f9f0f9f5c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/56e4c249006b460ba31357155a8beb72] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=36.6 K 2024-11-17T07:14:51,587 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting afaa8cf3f3784a8bbbb3c328c7e62981, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731827687973 2024-11-17T07:14:51,588 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 01fedcaa494f4c40bcc9423f9f0f9f5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1731827688774 2024-11-17T07:14:51,588 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 56e4c249006b460ba31357155a8beb72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731827689974 2024-11-17T07:14:51,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742147_1323 (size=14741) 2024-11-17T07:14:51,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/4070913c3c8f44a79c568c22876cc9da 2024-11-17T07:14:51,609 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#C#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:51,609 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/c9c14e50b5294726b24b52a6dd6b6872 is 50, key is test_row_0/C:col10/1731827690627/Put/seqid=0 2024-11-17T07:14:51,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/542707946609431f8bcd792cb8e45513 is 50, key is test_row_0/B:col10/1731827691549/Put/seqid=0 2024-11-17T07:14:51,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742149_1325 (size=12301) 2024-11-17T07:14:51,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742148_1324 (size=13017) 2024-11-17T07:14:51,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/542707946609431f8bcd792cb8e45513 2024-11-17T07:14:51,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827751623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:51,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:51,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:51,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:51,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/391bac75c9ae44eaac3d7616f6135105 is 50, key is test_row_0/C:col10/1731827691549/Put/seqid=0 2024-11-17T07:14:51,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742150_1326 (size=12301) 2024-11-17T07:14:51,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/391bac75c9ae44eaac3d7616f6135105 2024-11-17T07:14:51,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T07:14:51,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:51,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/4070913c3c8f44a79c568c22876cc9da as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4070913c3c8f44a79c568c22876cc9da 2024-11-17T07:14:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4070913c3c8f44a79c568c22876cc9da, entries=200, sequenceid=373, filesize=14.4 K 2024-11-17T07:14:51,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/542707946609431f8bcd792cb8e45513 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/542707946609431f8bcd792cb8e45513 2024-11-17T07:14:51,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/542707946609431f8bcd792cb8e45513, entries=150, sequenceid=373, filesize=12.0 K 2024-11-17T07:14:51,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/391bac75c9ae44eaac3d7616f6135105 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/391bac75c9ae44eaac3d7616f6135105 2024-11-17T07:14:51,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/391bac75c9ae44eaac3d7616f6135105, entries=150, sequenceid=373, filesize=12.0 K 2024-11-17T07:14:51,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 6d8c50823ce967d5944d0259cb160d17 in 117ms, sequenceid=373, compaction requested=false 2024-11-17T07:14:51,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:51,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T07:14:51,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:51,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:51,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:51,739 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:51,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:51,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:51,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/5f759d6d191b466f8efb6b06b75596c8 is 50, key is test_row_1/A:col10/1731827691611/Put/seqid=0 2024-11-17T07:14:51,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742151_1327 (size=12297) 2024-11-17T07:14:51,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/5f759d6d191b466f8efb6b06b75596c8 2024-11-17T07:14:51,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/3b48655f1bf1412badbecebdb3c79f72 is 50, key is test_row_1/B:col10/1731827691611/Put/seqid=0 2024-11-17T07:14:51,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742152_1328 (size=9857) 2024-11-17T07:14:51,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T07:14:51,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:51,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827751801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:51,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827751856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:51,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:51,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:51,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:51,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827751917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:51,921 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:51,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T07:14:51,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:51,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:51,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:51,960 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/3dfa84c035f84bc9b644abd27d256422 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3dfa84c035f84bc9b644abd27d256422 2024-11-17T07:14:51,966 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/A of 6d8c50823ce967d5944d0259cb160d17 into 3dfa84c035f84bc9b644abd27d256422(size=12.7 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:51,966 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:51,966 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/A, priority=13, startTime=1731827691499; duration=0sec 2024-11-17T07:14:51,967 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:51,967 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:A 2024-11-17T07:14:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T07:14:52,028 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/c9c14e50b5294726b24b52a6dd6b6872 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c9c14e50b5294726b24b52a6dd6b6872 2024-11-17T07:14:52,033 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/C of 6d8c50823ce967d5944d0259cb160d17 into c9c14e50b5294726b24b52a6dd6b6872(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:52,033 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:52,033 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/C, priority=13, startTime=1731827691499; duration=0sec 2024-11-17T07:14:52,034 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:52,034 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:C 2024-11-17T07:14:52,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T07:14:52,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:52,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:52,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:52,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:52,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:52,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827752127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:52,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:52,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/3b48655f1bf1412badbecebdb3c79f72 2024-11-17T07:14:52,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/fef1192605ec4b00867f4a871afdde50 is 50, key is test_row_1/C:col10/1731827691611/Put/seqid=0 2024-11-17T07:14:52,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742153_1329 (size=9857) 2024-11-17T07:14:52,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/fef1192605ec4b00867f4a871afdde50 2024-11-17T07:14:52,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/5f759d6d191b466f8efb6b06b75596c8 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5f759d6d191b466f8efb6b06b75596c8 2024-11-17T07:14:52,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5f759d6d191b466f8efb6b06b75596c8, entries=150, sequenceid=391, filesize=12.0 K 2024-11-17T07:14:52,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/3b48655f1bf1412badbecebdb3c79f72 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/3b48655f1bf1412badbecebdb3c79f72 2024-11-17T07:14:52,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/3b48655f1bf1412badbecebdb3c79f72, entries=100, sequenceid=391, filesize=9.6 K 2024-11-17T07:14:52,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/fef1192605ec4b00867f4a871afdde50 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/fef1192605ec4b00867f4a871afdde50 2024-11-17T07:14:52,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/fef1192605ec4b00867f4a871afdde50, entries=100, sequenceid=391, filesize=9.6 K 2024-11-17T07:14:52,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6d8c50823ce967d5944d0259cb160d17 in 505ms, sequenceid=391, compaction requested=true 2024-11-17T07:14:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:52,244 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:52,244 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:52,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:52,246 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:52,246 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/B is initiating minor compaction (all files) 2024-11-17T07:14:52,246 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/B in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:52,246 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/6794ec1beeb94459902da590a414e7ea, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/542707946609431f8bcd792cb8e45513, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/3b48655f1bf1412badbecebdb3c79f72] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=34.4 K 2024-11-17T07:14:52,246 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40055 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:52,247 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/A is initiating minor compaction (all files) 2024-11-17T07:14:52,247 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/A in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:52,247 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3dfa84c035f84bc9b644abd27d256422, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4070913c3c8f44a79c568c22876cc9da, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5f759d6d191b466f8efb6b06b75596c8] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=39.1 K 2024-11-17T07:14:52,248 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 6794ec1beeb94459902da590a414e7ea, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731827689974 2024-11-17T07:14:52,248 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dfa84c035f84bc9b644abd27d256422, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731827689974 2024-11-17T07:14:52,248 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 542707946609431f8bcd792cb8e45513, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731827690706 2024-11-17T07:14:52,248 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4070913c3c8f44a79c568c22876cc9da, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731827690645 2024-11-17T07:14:52,249 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b48655f1bf1412badbecebdb3c79f72, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731827691611 2024-11-17T07:14:52,249 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f759d6d191b466f8efb6b06b75596c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731827691611 2024-11-17T07:14:52,260 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#A#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:52,260 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/02a12b9524f84f6781f308b11c049dba is 50, key is test_row_0/A:col10/1731827691549/Put/seqid=0 2024-11-17T07:14:52,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-17T07:14:52,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:52,262 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T07:14:52,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:52,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:52,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:52,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:52,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:52,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:52,263 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#B#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:52,265 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/a4b1f000322b4093bfb932a3240336a7 is 50, key is test_row_0/B:col10/1731827691549/Put/seqid=0 2024-11-17T07:14:52,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/93d1b3ce44b242e29a5571e961f96ce7 is 50, key is test_row_0/A:col10/1731827691793/Put/seqid=0 2024-11-17T07:14:52,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742154_1330 (size=13119) 2024-11-17T07:14:52,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742155_1331 (size=13119) 2024-11-17T07:14:52,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742156_1332 (size=12301) 2024-11-17T07:14:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:52,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:52,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827752509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:52,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:52,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827752617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:52,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:52,683 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/02a12b9524f84f6781f308b11c049dba as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/02a12b9524f84f6781f308b11c049dba 2024-11-17T07:14:52,693 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/93d1b3ce44b242e29a5571e961f96ce7 2024-11-17T07:14:52,697 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/a4b1f000322b4093bfb932a3240336a7 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a4b1f000322b4093bfb932a3240336a7 2024-11-17T07:14:52,697 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/A of 6d8c50823ce967d5944d0259cb160d17 into 02a12b9524f84f6781f308b11c049dba(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:52,697 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:52,697 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/A, priority=13, startTime=1731827692244; duration=0sec 2024-11-17T07:14:52,698 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:52,698 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:A 2024-11-17T07:14:52,698 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:52,699 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:52,699 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/C is initiating minor compaction (all files) 2024-11-17T07:14:52,700 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/C in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:52,700 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c9c14e50b5294726b24b52a6dd6b6872, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/391bac75c9ae44eaac3d7616f6135105, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/fef1192605ec4b00867f4a871afdde50] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=34.4 K 2024-11-17T07:14:52,700 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9c14e50b5294726b24b52a6dd6b6872, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731827689974 2024-11-17T07:14:52,701 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 391bac75c9ae44eaac3d7616f6135105, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731827690706 2024-11-17T07:14:52,702 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting fef1192605ec4b00867f4a871afdde50, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731827691611 2024-11-17T07:14:52,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/2310569b73424880be118c0d0e61660b is 50, key is test_row_0/B:col10/1731827691793/Put/seqid=0 2024-11-17T07:14:52,703 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/B of 6d8c50823ce967d5944d0259cb160d17 into a4b1f000322b4093bfb932a3240336a7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:52,703 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:52,703 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/B, priority=13, startTime=1731827692244; duration=0sec 2024-11-17T07:14:52,703 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:52,703 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:B 2024-11-17T07:14:52,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742157_1333 (size=12301) 2024-11-17T07:14:52,710 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/2310569b73424880be118c0d0e61660b 2024-11-17T07:14:52,712 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#C#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:52,713 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/5a214580b2e94e629f8455c0ae4be26c is 50, key is test_row_0/C:col10/1731827691549/Put/seqid=0 2024-11-17T07:14:52,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742158_1334 (size=13119) 2024-11-17T07:14:52,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/f6e970870d134fe5ba19f2edbe4b3af7 is 50, key is test_row_0/C:col10/1731827691793/Put/seqid=0 2024-11-17T07:14:52,724 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/5a214580b2e94e629f8455c0ae4be26c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5a214580b2e94e629f8455c0ae4be26c 2024-11-17T07:14:52,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742159_1335 (size=12301) 2024-11-17T07:14:52,728 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/f6e970870d134fe5ba19f2edbe4b3af7 2024-11-17T07:14:52,730 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/C of 6d8c50823ce967d5944d0259cb160d17 into 5a214580b2e94e629f8455c0ae4be26c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:52,730 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:52,730 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/C, priority=13, startTime=1731827692244; duration=0sec 2024-11-17T07:14:52,731 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:52,731 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:C 2024-11-17T07:14:52,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/93d1b3ce44b242e29a5571e961f96ce7 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/93d1b3ce44b242e29a5571e961f96ce7 2024-11-17T07:14:52,737 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/93d1b3ce44b242e29a5571e961f96ce7, entries=150, sequenceid=408, filesize=12.0 K 2024-11-17T07:14:52,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/2310569b73424880be118c0d0e61660b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2310569b73424880be118c0d0e61660b 2024-11-17T07:14:52,741 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2310569b73424880be118c0d0e61660b, entries=150, sequenceid=408, filesize=12.0 K 2024-11-17T07:14:52,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/f6e970870d134fe5ba19f2edbe4b3af7 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/f6e970870d134fe5ba19f2edbe4b3af7 2024-11-17T07:14:52,745 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/f6e970870d134fe5ba19f2edbe4b3af7, entries=150, sequenceid=408, filesize=12.0 K 2024-11-17T07:14:52,746 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6d8c50823ce967d5944d0259cb160d17 in 484ms, sequenceid=408, compaction requested=false 2024-11-17T07:14:52,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:52,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:52,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-17T07:14:52,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-17T07:14:52,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-17T07:14:52,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8670 sec 2024-11-17T07:14:52,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.8720 sec 2024-11-17T07:14:52,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:52,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T07:14:52,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:52,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:52,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:52,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:52,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:52,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:52,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/0cca9d3b674d4578a83b92f1a07c6bff is 50, key is test_row_0/A:col10/1731827692828/Put/seqid=0 2024-11-17T07:14:52,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742160_1336 (size=14741) 2024-11-17T07:14:52,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:52,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827752877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:52,881 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in 6d8c50823ce967d5944d0259cb160d17 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:52,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:52,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:52,882 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:52,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:52,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827752881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:52,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:52,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:52,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:52,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827752986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:52,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:52,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-17T07:14:53,027 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:53,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-17T07:14:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T07:14:53,030 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:53,030 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:53,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:53,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T07:14:53,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T07:14:53,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:53,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:53,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:53,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:53,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:53,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:53,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827753197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:53,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/0cca9d3b674d4578a83b92f1a07c6bff 2024-11-17T07:14:53,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/4c6eb48c0ea54261b1ad57ca4cb9322b is 50, key is test_row_0/B:col10/1731827692828/Put/seqid=0 2024-11-17T07:14:53,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742161_1337 (size=12301) 2024-11-17T07:14:53,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/4c6eb48c0ea54261b1ad57ca4cb9322b 2024-11-17T07:14:53,274 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/41076083de0742a0812dcc21a015ac2c is 50, key is test_row_0/C:col10/1731827692828/Put/seqid=0 2024-11-17T07:14:53,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742162_1338 (size=12301) 2024-11-17T07:14:53,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T07:14:53,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:53,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:53,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:53,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:53,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:53,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T07:14:53,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T07:14:53,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:53,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:53,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:53,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:53,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:53,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:53,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827753507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:53,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T07:14:53,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:53,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:53,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:53,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:53,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:53,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T07:14:53,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/41076083de0742a0812dcc21a015ac2c 2024-11-17T07:14:53,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/0cca9d3b674d4578a83b92f1a07c6bff as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/0cca9d3b674d4578a83b92f1a07c6bff 2024-11-17T07:14:53,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/0cca9d3b674d4578a83b92f1a07c6bff, entries=200, sequenceid=429, filesize=14.4 K 2024-11-17T07:14:53,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/4c6eb48c0ea54261b1ad57ca4cb9322b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/4c6eb48c0ea54261b1ad57ca4cb9322b 2024-11-17T07:14:53,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/4c6eb48c0ea54261b1ad57ca4cb9322b, entries=150, sequenceid=429, filesize=12.0 K 2024-11-17T07:14:53,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/41076083de0742a0812dcc21a015ac2c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/41076083de0742a0812dcc21a015ac2c 2024-11-17T07:14:53,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/41076083de0742a0812dcc21a015ac2c, entries=150, sequenceid=429, filesize=12.0 K 2024-11-17T07:14:53,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6d8c50823ce967d5944d0259cb160d17 in 888ms, sequenceid=429, compaction requested=true 2024-11-17T07:14:53,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:53,717 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:53,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:53,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:53,718 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:53,718 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:53,719 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/A is initiating minor compaction (all files) 2024-11-17T07:14:53,719 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/A in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:53,719 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/02a12b9524f84f6781f308b11c049dba, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/93d1b3ce44b242e29a5571e961f96ce7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/0cca9d3b674d4578a83b92f1a07c6bff] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=39.2 K 2024-11-17T07:14:53,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:53,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:53,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:53,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:53,720 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02a12b9524f84f6781f308b11c049dba, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731827691549 2024-11-17T07:14:53,721 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:53,721 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/B is initiating minor compaction (all files) 2024-11-17T07:14:53,721 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/B in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:53,721 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a4b1f000322b4093bfb932a3240336a7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2310569b73424880be118c0d0e61660b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/4c6eb48c0ea54261b1ad57ca4cb9322b] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=36.8 K 2024-11-17T07:14:53,722 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93d1b3ce44b242e29a5571e961f96ce7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731827691780 2024-11-17T07:14:53,722 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a4b1f000322b4093bfb932a3240336a7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731827691549 2024-11-17T07:14:53,722 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cca9d3b674d4578a83b92f1a07c6bff, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731827692471 2024-11-17T07:14:53,722 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2310569b73424880be118c0d0e61660b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731827691780 2024-11-17T07:14:53,723 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c6eb48c0ea54261b1ad57ca4cb9322b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731827692488 2024-11-17T07:14:53,747 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#B#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:53,747 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/582246b680bf4d0b91aef50a3466532d is 50, key is test_row_0/B:col10/1731827692828/Put/seqid=0 2024-11-17T07:14:53,754 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#A#compaction#292 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:53,754 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/be6f5d8f7c49496fa7a9c2fe62242317 is 50, key is test_row_0/A:col10/1731827692828/Put/seqid=0 2024-11-17T07:14:53,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742163_1339 (size=13221) 2024-11-17T07:14:53,766 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/582246b680bf4d0b91aef50a3466532d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/582246b680bf4d0b91aef50a3466532d 2024-11-17T07:14:53,772 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/B of 6d8c50823ce967d5944d0259cb160d17 into 582246b680bf4d0b91aef50a3466532d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:53,772 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:53,772 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/B, priority=13, startTime=1731827693718; duration=0sec 2024-11-17T07:14:53,773 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:53,773 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:B 2024-11-17T07:14:53,773 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:53,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742164_1340 (size=13221) 2024-11-17T07:14:53,774 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:53,774 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/C is initiating minor compaction (all files) 2024-11-17T07:14:53,774 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/C in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:53,775 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5a214580b2e94e629f8455c0ae4be26c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/f6e970870d134fe5ba19f2edbe4b3af7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/41076083de0742a0812dcc21a015ac2c] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=36.8 K 2024-11-17T07:14:53,775 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a214580b2e94e629f8455c0ae4be26c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731827691549 2024-11-17T07:14:53,776 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting f6e970870d134fe5ba19f2edbe4b3af7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731827691780 2024-11-17T07:14:53,776 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 41076083de0742a0812dcc21a015ac2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731827692488 2024-11-17T07:14:53,780 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/be6f5d8f7c49496fa7a9c2fe62242317 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/be6f5d8f7c49496fa7a9c2fe62242317 2024-11-17T07:14:53,785 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#C#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:53,785 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/c0a7b4f6edd3406b8720b262a3520fd6 is 50, key is test_row_0/C:col10/1731827692828/Put/seqid=0 2024-11-17T07:14:53,789 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/A of 6d8c50823ce967d5944d0259cb160d17 into be6f5d8f7c49496fa7a9c2fe62242317(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:53,789 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:53,789 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/A, priority=13, startTime=1731827693717; duration=0sec 2024-11-17T07:14:53,789 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:53,789 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:A 2024-11-17T07:14:53,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-17T07:14:53,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:53,796 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T07:14:53,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:53,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:53,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:53,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:53,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:53,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:53,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742165_1341 (size=13221) 2024-11-17T07:14:53,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/7143137b24d944efa1d434c7f1c2c40d is 50, key is test_row_0/A:col10/1731827692879/Put/seqid=0 2024-11-17T07:14:53,822 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/c0a7b4f6edd3406b8720b262a3520fd6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c0a7b4f6edd3406b8720b262a3520fd6 2024-11-17T07:14:53,831 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/C of 6d8c50823ce967d5944d0259cb160d17 into c0a7b4f6edd3406b8720b262a3520fd6(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:53,831 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:53,831 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/C, priority=13, startTime=1731827693719; duration=0sec 2024-11-17T07:14:53,831 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:53,831 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:C 2024-11-17T07:14:53,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742166_1342 (size=9857) 2024-11-17T07:14:53,834 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/7143137b24d944efa1d434c7f1c2c40d 2024-11-17T07:14:53,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/54273f4ddfdd426eb5303e69fa8ea317 is 50, key is test_row_0/B:col10/1731827692879/Put/seqid=0 2024-11-17T07:14:53,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742167_1343 (size=9857) 2024-11-17T07:14:54,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:54,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:54,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:54,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827754095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T07:14:54,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:54,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827754207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:54,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:54,253 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/54273f4ddfdd426eb5303e69fa8ea317 2024-11-17T07:14:54,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/939d8e9643ae45a2a7fbd898f5d70503 is 50, key is test_row_0/C:col10/1731827692879/Put/seqid=0 2024-11-17T07:14:54,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742168_1344 (size=9857) 2024-11-17T07:14:54,324 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/939d8e9643ae45a2a7fbd898f5d70503 2024-11-17T07:14:54,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/7143137b24d944efa1d434c7f1c2c40d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/7143137b24d944efa1d434c7f1c2c40d 2024-11-17T07:14:54,356 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/7143137b24d944efa1d434c7f1c2c40d, entries=100, sequenceid=445, filesize=9.6 K 2024-11-17T07:14:54,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/54273f4ddfdd426eb5303e69fa8ea317 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/54273f4ddfdd426eb5303e69fa8ea317 2024-11-17T07:14:54,364 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/54273f4ddfdd426eb5303e69fa8ea317, entries=100, sequenceid=445, filesize=9.6 K 2024-11-17T07:14:54,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/939d8e9643ae45a2a7fbd898f5d70503 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/939d8e9643ae45a2a7fbd898f5d70503 2024-11-17T07:14:54,377 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/939d8e9643ae45a2a7fbd898f5d70503, entries=100, sequenceid=445, filesize=9.6 K 2024-11-17T07:14:54,379 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6d8c50823ce967d5944d0259cb160d17 in 582ms, sequenceid=445, compaction requested=false 2024-11-17T07:14:54,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:54,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:54,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-17T07:14:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-17T07:14:54,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-17T07:14:54,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3510 sec 2024-11-17T07:14:54,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.3550 sec 2024-11-17T07:14:54,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T07:14:54,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:54,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:54,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:54,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:54,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:54,421 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:54,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:54,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/1c81d7ce5c4742d3aa9b2be37a3a0d42 is 50, key is test_row_0/A:col10/1731827694094/Put/seqid=0 2024-11-17T07:14:54,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742169_1345 (size=14741) 2024-11-17T07:14:54,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/1c81d7ce5c4742d3aa9b2be37a3a0d42 2024-11-17T07:14:54,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/a538ce6a0ce4473f9d46660f1459d420 is 50, key is test_row_0/B:col10/1731827694094/Put/seqid=0 2024-11-17T07:14:54,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:54,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827754489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:54,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:54,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742170_1346 (size=12301) 2024-11-17T07:14:54,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/a538ce6a0ce4473f9d46660f1459d420 2024-11-17T07:14:54,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/7aa717de61264ea4a397f3a44a4d0dbe is 50, key is test_row_0/C:col10/1731827694094/Put/seqid=0 2024-11-17T07:14:54,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742171_1347 (size=12301) 2024-11-17T07:14:54,576 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/7aa717de61264ea4a397f3a44a4d0dbe 2024-11-17T07:14:54,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/1c81d7ce5c4742d3aa9b2be37a3a0d42 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1c81d7ce5c4742d3aa9b2be37a3a0d42 2024-11-17T07:14:54,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1c81d7ce5c4742d3aa9b2be37a3a0d42, entries=200, sequenceid=465, filesize=14.4 K 2024-11-17T07:14:54,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/a538ce6a0ce4473f9d46660f1459d420 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a538ce6a0ce4473f9d46660f1459d420 2024-11-17T07:14:54,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a538ce6a0ce4473f9d46660f1459d420, entries=150, sequenceid=465, filesize=12.0 K 2024-11-17T07:14:54,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/7aa717de61264ea4a397f3a44a4d0dbe as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/7aa717de61264ea4a397f3a44a4d0dbe 2024-11-17T07:14:54,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/7aa717de61264ea4a397f3a44a4d0dbe, entries=150, sequenceid=465, filesize=12.0 K 2024-11-17T07:14:54,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6d8c50823ce967d5944d0259cb160d17 in 178ms, sequenceid=465, compaction requested=true 2024-11-17T07:14:54,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:54,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:54,598 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:54,599 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:54,599 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:54,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:54,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:54,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:54,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:54,600 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:54,600 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37819 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:54,600 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/A is initiating minor compaction (all files) 2024-11-17T07:14:54,600 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/B is initiating minor compaction (all files) 2024-11-17T07:14:54,600 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/A in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:54,600 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/B in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:54,600 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/be6f5d8f7c49496fa7a9c2fe62242317, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/7143137b24d944efa1d434c7f1c2c40d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1c81d7ce5c4742d3aa9b2be37a3a0d42] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=36.9 K 2024-11-17T07:14:54,600 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/582246b680bf4d0b91aef50a3466532d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/54273f4ddfdd426eb5303e69fa8ea317, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a538ce6a0ce4473f9d46660f1459d420] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=34.5 K 2024-11-17T07:14:54,600 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting be6f5d8f7c49496fa7a9c2fe62242317, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731827692488 2024-11-17T07:14:54,600 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 582246b680bf4d0b91aef50a3466532d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731827692488 2024-11-17T07:14:54,601 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7143137b24d944efa1d434c7f1c2c40d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731827692868 2024-11-17T07:14:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:54,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T07:14:54,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:54,602 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 54273f4ddfdd426eb5303e69fa8ea317, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731827692868 2024-11-17T07:14:54,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:54,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:54,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:54,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:54,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:54,603 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c81d7ce5c4742d3aa9b2be37a3a0d42, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731827694078 2024-11-17T07:14:54,603 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a538ce6a0ce4473f9d46660f1459d420, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731827694092 2024-11-17T07:14:54,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/c744bb5aa43f45e38e980f0fd615ed82 is 50, key is test_row_0/A:col10/1731827694453/Put/seqid=0 2024-11-17T07:14:54,614 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#B#compaction#301 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:54,615 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/2b6e15a8d48e4636bfe01b1929a89203 is 50, key is test_row_0/B:col10/1731827694094/Put/seqid=0 2024-11-17T07:14:54,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742172_1348 (size=14741) 2024-11-17T07:14:54,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/c744bb5aa43f45e38e980f0fd615ed82 2024-11-17T07:14:54,638 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#A#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:54,639 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/cd615098da82436baf3ac7b1082a90aa is 50, key is test_row_0/A:col10/1731827694094/Put/seqid=0 2024-11-17T07:14:54,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/b274ae659a864ff3bd99b484e0f13400 is 50, key is test_row_0/B:col10/1731827694453/Put/seqid=0 2024-11-17T07:14:54,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742173_1349 (size=13323) 2024-11-17T07:14:54,670 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/2b6e15a8d48e4636bfe01b1929a89203 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2b6e15a8d48e4636bfe01b1929a89203 2024-11-17T07:14:54,676 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/B of 6d8c50823ce967d5944d0259cb160d17 into 2b6e15a8d48e4636bfe01b1929a89203(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:54,676 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:54,676 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/B, priority=13, startTime=1731827694599; duration=0sec 2024-11-17T07:14:54,676 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:54,676 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:B 2024-11-17T07:14:54,676 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:54,677 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:54,677 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/C is initiating minor compaction (all files) 2024-11-17T07:14:54,677 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/C in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:54,678 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c0a7b4f6edd3406b8720b262a3520fd6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/939d8e9643ae45a2a7fbd898f5d70503, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/7aa717de61264ea4a397f3a44a4d0dbe] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=34.5 K 2024-11-17T07:14:54,679 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting c0a7b4f6edd3406b8720b262a3520fd6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731827692488 2024-11-17T07:14:54,679 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 939d8e9643ae45a2a7fbd898f5d70503, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731827692868 2024-11-17T07:14:54,679 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aa717de61264ea4a397f3a44a4d0dbe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731827694092 2024-11-17T07:14:54,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742174_1350 (size=13323) 2024-11-17T07:14:54,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742175_1351 (size=12301) 2024-11-17T07:14:54,693 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#C#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:54,694 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/e3ad6add7d0444208707864ca4716240 is 50, key is test_row_0/C:col10/1731827694094/Put/seqid=0 2024-11-17T07:14:54,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827754690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:54,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:54,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:54,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:54,700 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/cd615098da82436baf3ac7b1082a90aa as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/cd615098da82436baf3ac7b1082a90aa 2024-11-17T07:14:54,706 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/A of 6d8c50823ce967d5944d0259cb160d17 into cd615098da82436baf3ac7b1082a90aa(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:54,706 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:54,706 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/A, priority=13, startTime=1731827694598; duration=0sec 2024-11-17T07:14:54,706 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:54,706 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:A 2024-11-17T07:14:54,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742176_1352 (size=13323) 2024-11-17T07:14:54,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827754807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:54,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41686 deadline: 1731827754907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:54,910 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in 6d8c50823ce967d5944d0259cb160d17 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:14:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:54,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:55,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827755017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:55,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:55,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/b274ae659a864ff3bd99b484e0f13400 2024-11-17T07:14:55,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/de361339e1b84f6db5c770f7c52322d9 is 50, key is test_row_0/C:col10/1731827694453/Put/seqid=0 2024-11-17T07:14:55,119 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/e3ad6add7d0444208707864ca4716240 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e3ad6add7d0444208707864ca4716240 2024-11-17T07:14:55,132 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/C of 6d8c50823ce967d5944d0259cb160d17 into e3ad6add7d0444208707864ca4716240(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:55,132 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:55,132 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/C, priority=13, startTime=1731827694599; duration=0sec 2024-11-17T07:14:55,132 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:55,133 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:C 2024-11-17T07:14:55,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742177_1353 (size=12301) 2024-11-17T07:14:55,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/de361339e1b84f6db5c770f7c52322d9 2024-11-17T07:14:55,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/c744bb5aa43f45e38e980f0fd615ed82 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/c744bb5aa43f45e38e980f0fd615ed82 2024-11-17T07:14:55,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/c744bb5aa43f45e38e980f0fd615ed82, entries=200, sequenceid=479, filesize=14.4 K 2024-11-17T07:14:55,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/b274ae659a864ff3bd99b484e0f13400 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/b274ae659a864ff3bd99b484e0f13400 2024-11-17T07:14:55,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-17T07:14:55,178 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:55,179 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:14:55,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-17T07:14:55,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/b274ae659a864ff3bd99b484e0f13400, entries=150, sequenceid=479, filesize=12.0 K 2024-11-17T07:14:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/de361339e1b84f6db5c770f7c52322d9 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/de361339e1b84f6db5c770f7c52322d9 2024-11-17T07:14:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T07:14:55,182 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:14:55,183 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:14:55,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:14:55,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/de361339e1b84f6db5c770f7c52322d9, entries=150, sequenceid=479, filesize=12.0 K 2024-11-17T07:14:55,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6d8c50823ce967d5944d0259cb160d17 in 584ms, sequenceid=479, compaction requested=false 2024-11-17T07:14:55,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:55,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:55,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T07:14:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:55,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:55,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/6e155a3f95eb45c88c5b21d476faa999 is 50, key is test_row_0/A:col10/1731827695188/Put/seqid=0 2024-11-17T07:14:55,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:55,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827755257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:55,265 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:55,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742178_1354 (size=14741) 2024-11-17T07:14:55,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/6e155a3f95eb45c88c5b21d476faa999 2024-11-17T07:14:55,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:55,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827755266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:55,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T07:14:55,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/e812e49dae5943c392e06bad4526fbcd is 50, key is test_row_0/B:col10/1731827695188/Put/seqid=0 2024-11-17T07:14:55,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:55,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41674 deadline: 1731827755328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:55,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:55,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742179_1355 (size=12301) 2024-11-17T07:14:55,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/e812e49dae5943c392e06bad4526fbcd 2024-11-17T07:14:55,337 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-17T07:14:55,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:55,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:55,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:55,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:55,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:55,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:14:55,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/9d8f6147b30e42d4b72498371be55638 is 50, key is test_row_0/C:col10/1731827695188/Put/seqid=0 2024-11-17T07:14:55,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41622 deadline: 1731827755367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:55,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:55,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:14:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41678 deadline: 1731827755377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:14:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:14:55,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742180_1356 (size=12301) 2024-11-17T07:14:55,418 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/9d8f6147b30e42d4b72498371be55638 2024-11-17T07:14:55,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/6e155a3f95eb45c88c5b21d476faa999 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/6e155a3f95eb45c88c5b21d476faa999 2024-11-17T07:14:55,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/6e155a3f95eb45c88c5b21d476faa999, entries=200, sequenceid=501, filesize=14.4 K 2024-11-17T07:14:55,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/e812e49dae5943c392e06bad4526fbcd as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/e812e49dae5943c392e06bad4526fbcd 2024-11-17T07:14:55,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/e812e49dae5943c392e06bad4526fbcd, entries=150, sequenceid=501, filesize=12.0 K 2024-11-17T07:14:55,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/9d8f6147b30e42d4b72498371be55638 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/9d8f6147b30e42d4b72498371be55638 2024-11-17T07:14:55,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/9d8f6147b30e42d4b72498371be55638, entries=150, sequenceid=501, filesize=12.0 K 2024-11-17T07:14:55,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6d8c50823ce967d5944d0259cb160d17 in 257ms, sequenceid=501, compaction requested=true 2024-11-17T07:14:55,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:55,447 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:55,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:14:55,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:55,448 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:55,448 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42805 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:55,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:14:55,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:55,449 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/A is initiating minor compaction (all files) 2024-11-17T07:14:55,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d8c50823ce967d5944d0259cb160d17:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:14:55,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:55,449 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/A in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:55,449 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/cd615098da82436baf3ac7b1082a90aa, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/c744bb5aa43f45e38e980f0fd615ed82, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/6e155a3f95eb45c88c5b21d476faa999] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=41.8 K 2024-11-17T07:14:55,450 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd615098da82436baf3ac7b1082a90aa, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731827694092 2024-11-17T07:14:55,451 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:55,451 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/B is initiating minor compaction (all files) 2024-11-17T07:14:55,451 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/B in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:55,451 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2b6e15a8d48e4636bfe01b1929a89203, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/b274ae659a864ff3bd99b484e0f13400, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/e812e49dae5943c392e06bad4526fbcd] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=37.0 K 2024-11-17T07:14:55,452 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting c744bb5aa43f45e38e980f0fd615ed82, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731827694453 2024-11-17T07:14:55,452 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b6e15a8d48e4636bfe01b1929a89203, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731827694092 2024-11-17T07:14:55,454 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e155a3f95eb45c88c5b21d476faa999, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731827694667 2024-11-17T07:14:55,454 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting b274ae659a864ff3bd99b484e0f13400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731827694453 2024-11-17T07:14:55,455 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting e812e49dae5943c392e06bad4526fbcd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731827694667 2024-11-17T07:14:55,477 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#B#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:55,478 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/cc9e421406954c679ed63ffc17ea804c is 50, key is test_row_0/B:col10/1731827695188/Put/seqid=0 2024-11-17T07:14:55,481 INFO [Thread-1200 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1200. 2024-11-17T07:14:55,481 DEBUG [Thread-1200 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:55,481 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,482 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:55,483 INFO [Thread-1202 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1202. 2024-11-17T07:14:55,483 DEBUG [Thread-1202 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:55,483 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,483 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:55,485 INFO [Thread-1198 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1198. 2024-11-17T07:14:55,485 DEBUG [Thread-1198 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:55,486 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,486 INFO [Thread-1204 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1204. 2024-11-17T07:14:55,486 DEBUG [Thread-1204 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:55,486 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,486 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:55,486 INFO [Thread-1196 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1196. 2024-11-17T07:14:55,486 DEBUG [Thread-1196 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:55,486 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:55,486 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,486 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,486 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:55,489 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#A#compaction#310 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:55,489 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/605c1686f5d14a638dd8e9a117cfe5e1 is 50, key is test_row_0/A:col10/1731827695188/Put/seqid=0 2024-11-17T07:14:55,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-17T07:14:55,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:55,491 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T07:14:55,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:55,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:55,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:55,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:55,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:55,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:55,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742181_1357 (size=13425) 2024-11-17T07:14:55,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T07:14:55,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/4b83bcc96c1a4d608c01dc133fde38db is 50, key is test_row_0/A:col10/1731827695262/Put/seqid=0 2024-11-17T07:14:55,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742182_1358 (size=13425) 2024-11-17T07:14:55,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742183_1359 (size=12301) 2024-11-17T07:14:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:55,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. as already flushing 2024-11-17T07:14:55,579 INFO [Thread-1189 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1189. 2024-11-17T07:14:55,579 DEBUG [Thread-1189 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:55,580 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,580 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:55,588 INFO [Thread-1191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1191. 2024-11-17T07:14:55,589 DEBUG [Thread-1191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:55,589 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,589 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T07:14:55,839 INFO [Thread-1187 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1187. 2024-11-17T07:14:55,839 DEBUG [Thread-1187 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:55,839 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,839 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:55,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:55,903 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/cc9e421406954c679ed63ffc17ea804c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/cc9e421406954c679ed63ffc17ea804c 2024-11-17T07:14:55,909 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/B of 6d8c50823ce967d5944d0259cb160d17 into cc9e421406954c679ed63ffc17ea804c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:55,909 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:55,909 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/B, priority=13, startTime=1731827695448; duration=0sec 2024-11-17T07:14:55,909 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:14:55,910 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:B 2024-11-17T07:14:55,910 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:14:55,911 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:14:55,911 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): 6d8c50823ce967d5944d0259cb160d17/C is initiating minor compaction (all files) 2024-11-17T07:14:55,911 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d8c50823ce967d5944d0259cb160d17/C in TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:55,911 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e3ad6add7d0444208707864ca4716240, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/de361339e1b84f6db5c770f7c52322d9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/9d8f6147b30e42d4b72498371be55638] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp, totalSize=37.0 K 2024-11-17T07:14:55,911 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting e3ad6add7d0444208707864ca4716240, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1731827694092 2024-11-17T07:14:55,912 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting de361339e1b84f6db5c770f7c52322d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=479, earliestPutTs=1731827694453 2024-11-17T07:14:55,912 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d8f6147b30e42d4b72498371be55638, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731827694667 2024-11-17T07:14:55,927 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d8c50823ce967d5944d0259cb160d17#C#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:14:55,927 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/e552d754d01041f1882a377f8b3def86 is 50, key is test_row_0/C:col10/1731827695188/Put/seqid=0 2024-11-17T07:14:55,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742184_1360 (size=13425) 2024-11-17T07:14:55,949 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/4b83bcc96c1a4d608c01dc133fde38db 2024-11-17T07:14:55,955 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/605c1686f5d14a638dd8e9a117cfe5e1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/605c1686f5d14a638dd8e9a117cfe5e1 2024-11-17T07:14:55,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/df2cea28235e4ed9bdbba9596f33c8f6 is 50, key is test_row_0/B:col10/1731827695262/Put/seqid=0 2024-11-17T07:14:55,961 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/A of 6d8c50823ce967d5944d0259cb160d17 into 605c1686f5d14a638dd8e9a117cfe5e1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:55,961 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:55,961 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/A, priority=13, startTime=1731827695447; duration=0sec 2024-11-17T07:14:55,961 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:55,961 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:A 2024-11-17T07:14:55,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742185_1361 (size=12301) 2024-11-17T07:14:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T07:14:56,340 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/e552d754d01041f1882a377f8b3def86 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e552d754d01041f1882a377f8b3def86 2024-11-17T07:14:56,343 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d8c50823ce967d5944d0259cb160d17/C of 6d8c50823ce967d5944d0259cb160d17 into e552d754d01041f1882a377f8b3def86(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:14:56,344 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:56,344 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., storeName=6d8c50823ce967d5944d0259cb160d17/C, priority=13, startTime=1731827695449; duration=0sec 2024-11-17T07:14:56,344 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:14:56,344 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d8c50823ce967d5944d0259cb160d17:C 2024-11-17T07:14:56,370 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/df2cea28235e4ed9bdbba9596f33c8f6 2024-11-17T07:14:56,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/c6ecffd652164bb59f1de351099b42bc is 50, key is test_row_0/C:col10/1731827695262/Put/seqid=0 2024-11-17T07:14:56,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742186_1362 (size=12301) 2024-11-17T07:14:56,780 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/c6ecffd652164bb59f1de351099b42bc 2024-11-17T07:14:56,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/4b83bcc96c1a4d608c01dc133fde38db as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4b83bcc96c1a4d608c01dc133fde38db 2024-11-17T07:14:56,787 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4b83bcc96c1a4d608c01dc133fde38db, entries=150, sequenceid=514, filesize=12.0 K 2024-11-17T07:14:56,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/df2cea28235e4ed9bdbba9596f33c8f6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/df2cea28235e4ed9bdbba9596f33c8f6 2024-11-17T07:14:56,791 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/df2cea28235e4ed9bdbba9596f33c8f6, entries=150, sequenceid=514, filesize=12.0 K 2024-11-17T07:14:56,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/c6ecffd652164bb59f1de351099b42bc as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c6ecffd652164bb59f1de351099b42bc 2024-11-17T07:14:56,794 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c6ecffd652164bb59f1de351099b42bc, entries=150, sequenceid=514, filesize=12.0 K 2024-11-17T07:14:56,795 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=20.13 KB/20610 for 6d8c50823ce967d5944d0259cb160d17 in 1304ms, sequenceid=514, compaction requested=false 2024-11-17T07:14:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 6d8c50823ce967d5944d0259cb160d17: 2024-11-17T07:14:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:56,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-17T07:14:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-17T07:14:56,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-17T07:14:56,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6130 sec 2024-11-17T07:14:56,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 1.6190 sec 2024-11-17T07:14:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-17T07:14:57,328 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:14:58,948 INFO [Thread-1193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1193. 2024-11-17T07:14:58,948 DEBUG [Thread-1193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:58,948 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:58,949 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:58,949 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:59,329 INFO [Thread-1185 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1185. 2024-11-17T07:14:59,329 DEBUG [Thread-1185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:14:59,329 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:59,329 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:59,329 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:59,329 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T07:14:59,329 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-17T07:14:59,329 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 151 2024-11-17T07:14:59,329 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-17T07:14:59,329 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-17T07:14:59,330 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 176 2024-11-17T07:14:59,330 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T07:14:59,330 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3173 2024-11-17T07:14:59,330 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3014 2024-11-17T07:14:59,330 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2975 2024-11-17T07:14:59,330 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3183 2024-11-17T07:14:59,330 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3028 2024-11-17T07:14:59,330 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T07:14:59,330 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T07:14:59,330 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T07:14:59,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:59,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:14:59,330 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:14:59,331 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T07:14:59,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T07:14:59,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T07:14:59,333 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827699333"}]},"ts":"1731827699333"} 2024-11-17T07:14:59,334 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T07:14:59,334 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T07:14:59,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T07:14:59,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d8c50823ce967d5944d0259cb160d17, UNASSIGN}] 2024-11-17T07:14:59,337 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d8c50823ce967d5944d0259cb160d17, UNASSIGN 2024-11-17T07:14:59,337 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=6d8c50823ce967d5944d0259cb160d17, regionState=CLOSING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:14:59,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d8c50823ce967d5944d0259cb160d17, UNASSIGN because future has completed 2024-11-17T07:14:59,339 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T07:14:59,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:14:59,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T07:14:59,489 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:14:59,489 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T07:14:59,490 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 6d8c50823ce967d5944d0259cb160d17, disabling compactions & flushes 2024-11-17T07:14:59,490 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:59,490 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:59,490 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. after waiting 0 ms 2024-11-17T07:14:59,490 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:14:59,490 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing 6d8c50823ce967d5944d0259cb160d17 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T07:14:59,490 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=A 2024-11-17T07:14:59,490 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:59,490 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=B 2024-11-17T07:14:59,490 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:59,490 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6d8c50823ce967d5944d0259cb160d17, store=C 2024-11-17T07:14:59,490 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:14:59,494 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/9ceab27dd1304a7db060ec46e1c1de31 is 50, key is test_row_0/A:col10/1731827699328/Put/seqid=0 2024-11-17T07:14:59,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742187_1363 (size=12301) 2024-11-17T07:14:59,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T07:14:59,897 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/9ceab27dd1304a7db060ec46e1c1de31 2024-11-17T07:14:59,903 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/1fe44e0025924e80b31ea6860b5b23e9 is 50, key is test_row_0/B:col10/1731827699328/Put/seqid=0 2024-11-17T07:14:59,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742188_1364 (size=12301) 2024-11-17T07:14:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T07:15:00,307 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/1fe44e0025924e80b31ea6860b5b23e9 2024-11-17T07:15:00,313 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/e2cb00b35aad4e8386faddb93882bdae is 50, key is test_row_0/C:col10/1731827699328/Put/seqid=0 2024-11-17T07:15:00,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742189_1365 (size=12301) 2024-11-17T07:15:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T07:15:00,716 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/e2cb00b35aad4e8386faddb93882bdae 2024-11-17T07:15:00,720 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/A/9ceab27dd1304a7db060ec46e1c1de31 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/9ceab27dd1304a7db060ec46e1c1de31 2024-11-17T07:15:00,723 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/9ceab27dd1304a7db060ec46e1c1de31, entries=150, sequenceid=525, filesize=12.0 K 2024-11-17T07:15:00,724 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/B/1fe44e0025924e80b31ea6860b5b23e9 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/1fe44e0025924e80b31ea6860b5b23e9 2024-11-17T07:15:00,727 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/1fe44e0025924e80b31ea6860b5b23e9, entries=150, sequenceid=525, filesize=12.0 K 2024-11-17T07:15:00,727 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/.tmp/C/e2cb00b35aad4e8386faddb93882bdae as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e2cb00b35aad4e8386faddb93882bdae 2024-11-17T07:15:00,730 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e2cb00b35aad4e8386faddb93882bdae, entries=150, sequenceid=525, filesize=12.0 K 2024-11-17T07:15:00,731 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6d8c50823ce967d5944d0259cb160d17 in 1241ms, sequenceid=525, compaction requested=true 2024-11-17T07:15:00,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/ef429386e49843eba6b6532ce81db6c5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/d41dfe8443964a33b112f3ebbdf86b87, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1f1f6d5b5c6146b282904d945adb2a3a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5c58245a9a87446f98280b1f34b02bfa, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/b715ce85a4cc43b79845a79cae95cfe7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5f910ea163da445e8f53c7adad440955, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/8ae2778c4b7a499f9c6ce8733bc0b53a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/6ab0a23dea90435e83fba330111f03ec, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/a96e0343b9a646f5be257bed3a4c126f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/7e28f628ad0746508c3ce0f91cd93a60, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4b49d8c5ce4447578b4a59ff58d7db2b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/b505bfd7b5b041268c8095d7b46b58dd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3e3da9bbca3d46358917858c4c3b84d0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/8c36010c586c43d9a5da7630db0188aa, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3864c2cf1b9a4500906dfc79ae56f595, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/05ee820ad54d40138c0aa9a85e1c49d6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/ed6117189e2e467c821a0eff249c0f75, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/632f5246ff544083a108367daaabc08e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/9363dffb45f54f00afae025fc94f184c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/debf283689674ba89ee9bb3c2e72c4e3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3dfa84c035f84bc9b644abd27d256422, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1e4ea9497a324c0c95abae7211ad433a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4070913c3c8f44a79c568c22876cc9da, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/02a12b9524f84f6781f308b11c049dba, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5f759d6d191b466f8efb6b06b75596c8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/93d1b3ce44b242e29a5571e961f96ce7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/0cca9d3b674d4578a83b92f1a07c6bff, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/be6f5d8f7c49496fa7a9c2fe62242317, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/7143137b24d944efa1d434c7f1c2c40d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1c81d7ce5c4742d3aa9b2be37a3a0d42, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/cd615098da82436baf3ac7b1082a90aa, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/c744bb5aa43f45e38e980f0fd615ed82, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/6e155a3f95eb45c88c5b21d476faa999] to archive 2024-11-17T07:15:00,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:15:00,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/ef429386e49843eba6b6532ce81db6c5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/ef429386e49843eba6b6532ce81db6c5 2024-11-17T07:15:00,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/d41dfe8443964a33b112f3ebbdf86b87 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/d41dfe8443964a33b112f3ebbdf86b87 2024-11-17T07:15:00,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1f1f6d5b5c6146b282904d945adb2a3a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1f1f6d5b5c6146b282904d945adb2a3a 2024-11-17T07:15:00,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5c58245a9a87446f98280b1f34b02bfa to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5c58245a9a87446f98280b1f34b02bfa 2024-11-17T07:15:00,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/b715ce85a4cc43b79845a79cae95cfe7 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/b715ce85a4cc43b79845a79cae95cfe7 2024-11-17T07:15:00,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5f910ea163da445e8f53c7adad440955 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5f910ea163da445e8f53c7adad440955 2024-11-17T07:15:00,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/8ae2778c4b7a499f9c6ce8733bc0b53a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/8ae2778c4b7a499f9c6ce8733bc0b53a 2024-11-17T07:15:00,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/6ab0a23dea90435e83fba330111f03ec to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/6ab0a23dea90435e83fba330111f03ec 2024-11-17T07:15:00,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/a96e0343b9a646f5be257bed3a4c126f to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/a96e0343b9a646f5be257bed3a4c126f 2024-11-17T07:15:00,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/7e28f628ad0746508c3ce0f91cd93a60 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/7e28f628ad0746508c3ce0f91cd93a60 2024-11-17T07:15:00,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4b49d8c5ce4447578b4a59ff58d7db2b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4b49d8c5ce4447578b4a59ff58d7db2b 2024-11-17T07:15:00,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/b505bfd7b5b041268c8095d7b46b58dd to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/b505bfd7b5b041268c8095d7b46b58dd 2024-11-17T07:15:00,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3e3da9bbca3d46358917858c4c3b84d0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3e3da9bbca3d46358917858c4c3b84d0 2024-11-17T07:15:00,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/8c36010c586c43d9a5da7630db0188aa to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/8c36010c586c43d9a5da7630db0188aa 2024-11-17T07:15:00,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3864c2cf1b9a4500906dfc79ae56f595 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3864c2cf1b9a4500906dfc79ae56f595 2024-11-17T07:15:00,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/05ee820ad54d40138c0aa9a85e1c49d6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/05ee820ad54d40138c0aa9a85e1c49d6 2024-11-17T07:15:00,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/ed6117189e2e467c821a0eff249c0f75 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/ed6117189e2e467c821a0eff249c0f75 2024-11-17T07:15:00,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/632f5246ff544083a108367daaabc08e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/632f5246ff544083a108367daaabc08e 2024-11-17T07:15:00,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/9363dffb45f54f00afae025fc94f184c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/9363dffb45f54f00afae025fc94f184c 2024-11-17T07:15:00,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/debf283689674ba89ee9bb3c2e72c4e3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/debf283689674ba89ee9bb3c2e72c4e3 2024-11-17T07:15:00,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3dfa84c035f84bc9b644abd27d256422 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/3dfa84c035f84bc9b644abd27d256422 2024-11-17T07:15:00,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1e4ea9497a324c0c95abae7211ad433a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1e4ea9497a324c0c95abae7211ad433a 2024-11-17T07:15:00,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4070913c3c8f44a79c568c22876cc9da to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4070913c3c8f44a79c568c22876cc9da 2024-11-17T07:15:00,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/02a12b9524f84f6781f308b11c049dba to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/02a12b9524f84f6781f308b11c049dba 2024-11-17T07:15:00,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5f759d6d191b466f8efb6b06b75596c8 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/5f759d6d191b466f8efb6b06b75596c8 2024-11-17T07:15:00,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/93d1b3ce44b242e29a5571e961f96ce7 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/93d1b3ce44b242e29a5571e961f96ce7 2024-11-17T07:15:00,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/0cca9d3b674d4578a83b92f1a07c6bff to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/0cca9d3b674d4578a83b92f1a07c6bff 2024-11-17T07:15:00,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/be6f5d8f7c49496fa7a9c2fe62242317 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/be6f5d8f7c49496fa7a9c2fe62242317 2024-11-17T07:15:00,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/7143137b24d944efa1d434c7f1c2c40d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/7143137b24d944efa1d434c7f1c2c40d 2024-11-17T07:15:00,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1c81d7ce5c4742d3aa9b2be37a3a0d42 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/1c81d7ce5c4742d3aa9b2be37a3a0d42 2024-11-17T07:15:00,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/cd615098da82436baf3ac7b1082a90aa to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/cd615098da82436baf3ac7b1082a90aa 2024-11-17T07:15:00,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/c744bb5aa43f45e38e980f0fd615ed82 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/c744bb5aa43f45e38e980f0fd615ed82 2024-11-17T07:15:00,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/6e155a3f95eb45c88c5b21d476faa999 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/6e155a3f95eb45c88c5b21d476faa999 2024-11-17T07:15:00,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/0b7914197e654db882ecb46ec41f2420, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2c245a1c4ff9485f9853c8b9a9ee94db, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/dd9264361de04c49a200cf1332ad2026, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/326a859ee39d4dee80c11417199ade3d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/43d12f86aaa2470da60cc5ae1785b375, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/5229a4f1e10549b8bfbadda157ffe507, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/6b8ee3f9fd77416fa3cd488902377b36, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/8f8fc5e90623443b867a7e8b51692717, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/999a080ff43146c8a7d4bbb49d673582, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/84922ca228a14149bce6379206218ad3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a907b5a95c7f4d398fdd1c46a50c78e1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/90982646fefd47d2b63fec4658dda753, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/77c08d044cfd4a79935719c7815c06ed, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/46a247653de94a9793b83b246f877444, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/33d5944538b94c16ba757cfd0a7841bb, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/de3fda9427254430a120ed4fcbd6b8de, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/f088c1eee479457b8e0c7660454585c0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/bd1abd38a59349fb91df7fcecff9072a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/ff193940c4a5481686a5bd718b33fe90, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/5fba3ee25bc44ace92259bd9de57e2a0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/6794ec1beeb94459902da590a414e7ea, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/7dce06921b2248a9bc8ce0710d95549d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/542707946609431f8bcd792cb8e45513, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a4b1f000322b4093bfb932a3240336a7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/3b48655f1bf1412badbecebdb3c79f72, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2310569b73424880be118c0d0e61660b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/582246b680bf4d0b91aef50a3466532d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/4c6eb48c0ea54261b1ad57ca4cb9322b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/54273f4ddfdd426eb5303e69fa8ea317, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2b6e15a8d48e4636bfe01b1929a89203, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a538ce6a0ce4473f9d46660f1459d420, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/b274ae659a864ff3bd99b484e0f13400, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/e812e49dae5943c392e06bad4526fbcd] to archive 2024-11-17T07:15:00,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:15:00,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/0b7914197e654db882ecb46ec41f2420 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/0b7914197e654db882ecb46ec41f2420 2024-11-17T07:15:00,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2c245a1c4ff9485f9853c8b9a9ee94db to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2c245a1c4ff9485f9853c8b9a9ee94db 2024-11-17T07:15:00,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/dd9264361de04c49a200cf1332ad2026 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/dd9264361de04c49a200cf1332ad2026 2024-11-17T07:15:00,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/326a859ee39d4dee80c11417199ade3d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/326a859ee39d4dee80c11417199ade3d 2024-11-17T07:15:00,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/43d12f86aaa2470da60cc5ae1785b375 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/43d12f86aaa2470da60cc5ae1785b375 2024-11-17T07:15:00,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/5229a4f1e10549b8bfbadda157ffe507 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/5229a4f1e10549b8bfbadda157ffe507 2024-11-17T07:15:00,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/6b8ee3f9fd77416fa3cd488902377b36 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/6b8ee3f9fd77416fa3cd488902377b36 2024-11-17T07:15:00,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/8f8fc5e90623443b867a7e8b51692717 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/8f8fc5e90623443b867a7e8b51692717 2024-11-17T07:15:00,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/999a080ff43146c8a7d4bbb49d673582 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/999a080ff43146c8a7d4bbb49d673582 2024-11-17T07:15:00,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/84922ca228a14149bce6379206218ad3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/84922ca228a14149bce6379206218ad3 2024-11-17T07:15:00,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a907b5a95c7f4d398fdd1c46a50c78e1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a907b5a95c7f4d398fdd1c46a50c78e1 2024-11-17T07:15:00,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/90982646fefd47d2b63fec4658dda753 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/90982646fefd47d2b63fec4658dda753 2024-11-17T07:15:00,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/77c08d044cfd4a79935719c7815c06ed to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/77c08d044cfd4a79935719c7815c06ed 2024-11-17T07:15:00,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/46a247653de94a9793b83b246f877444 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/46a247653de94a9793b83b246f877444 2024-11-17T07:15:00,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/33d5944538b94c16ba757cfd0a7841bb to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/33d5944538b94c16ba757cfd0a7841bb 2024-11-17T07:15:00,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/de3fda9427254430a120ed4fcbd6b8de to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/de3fda9427254430a120ed4fcbd6b8de 2024-11-17T07:15:00,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/f088c1eee479457b8e0c7660454585c0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/f088c1eee479457b8e0c7660454585c0 2024-11-17T07:15:00,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/bd1abd38a59349fb91df7fcecff9072a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/bd1abd38a59349fb91df7fcecff9072a 2024-11-17T07:15:00,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/ff193940c4a5481686a5bd718b33fe90 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/ff193940c4a5481686a5bd718b33fe90 2024-11-17T07:15:00,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/5fba3ee25bc44ace92259bd9de57e2a0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/5fba3ee25bc44ace92259bd9de57e2a0 2024-11-17T07:15:00,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/6794ec1beeb94459902da590a414e7ea to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/6794ec1beeb94459902da590a414e7ea 2024-11-17T07:15:00,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/7dce06921b2248a9bc8ce0710d95549d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/7dce06921b2248a9bc8ce0710d95549d 2024-11-17T07:15:00,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/542707946609431f8bcd792cb8e45513 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/542707946609431f8bcd792cb8e45513 2024-11-17T07:15:00,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a4b1f000322b4093bfb932a3240336a7 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a4b1f000322b4093bfb932a3240336a7 2024-11-17T07:15:00,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/3b48655f1bf1412badbecebdb3c79f72 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/3b48655f1bf1412badbecebdb3c79f72 2024-11-17T07:15:00,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2310569b73424880be118c0d0e61660b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2310569b73424880be118c0d0e61660b 2024-11-17T07:15:00,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/582246b680bf4d0b91aef50a3466532d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/582246b680bf4d0b91aef50a3466532d 2024-11-17T07:15:00,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/4c6eb48c0ea54261b1ad57ca4cb9322b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/4c6eb48c0ea54261b1ad57ca4cb9322b 2024-11-17T07:15:00,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/54273f4ddfdd426eb5303e69fa8ea317 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/54273f4ddfdd426eb5303e69fa8ea317 2024-11-17T07:15:00,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2b6e15a8d48e4636bfe01b1929a89203 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/2b6e15a8d48e4636bfe01b1929a89203 2024-11-17T07:15:00,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a538ce6a0ce4473f9d46660f1459d420 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/a538ce6a0ce4473f9d46660f1459d420 2024-11-17T07:15:00,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/b274ae659a864ff3bd99b484e0f13400 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/b274ae659a864ff3bd99b484e0f13400 2024-11-17T07:15:00,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/e812e49dae5943c392e06bad4526fbcd to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/e812e49dae5943c392e06bad4526fbcd 2024-11-17T07:15:00,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/14addfc3568d4a1691424c71c49a6019, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5da112d639a841f8aff096330cb3acd6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d764b5aa4d8048baa37cdb0b1e3cc582, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d1b3725bb25f4ee8ac30575b19f92927, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/ca1a58b9920d4a62865cd48f4d50635a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e18ff74a638c49cabeeef00de606fee4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d486a7186bec4d1e9f0d6cc3bfbf9843, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5df402122ebf44e0b477eb8613f1b7f0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/539c7d6eccde4a15a2b021a0cc7698d5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e176277b329f4ac48009c5f52dd0e2ce, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/9e2802f856634dcdb85b7afa323262c0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/397c5dda280e4451a38a932faab1742f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/2e12dbf5e7364d7e825361288db63bf1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/6705e0b68ef143afa204fb22b4d8c2d0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/167bc983ca1844b397d3eaa0da2c2a23, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/06672b322a424068adf3dbe163d56209, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/781192e106f3466e9e63aa95b4758a7b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/afaa8cf3f3784a8bbbb3c328c7e62981, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/cbfecdaa1b364d12845b56ad03a42bf1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/01fedcaa494f4c40bcc9423f9f0f9f5c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c9c14e50b5294726b24b52a6dd6b6872, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/56e4c249006b460ba31357155a8beb72, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/391bac75c9ae44eaac3d7616f6135105, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5a214580b2e94e629f8455c0ae4be26c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/fef1192605ec4b00867f4a871afdde50, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/f6e970870d134fe5ba19f2edbe4b3af7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c0a7b4f6edd3406b8720b262a3520fd6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/41076083de0742a0812dcc21a015ac2c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/939d8e9643ae45a2a7fbd898f5d70503, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e3ad6add7d0444208707864ca4716240, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/7aa717de61264ea4a397f3a44a4d0dbe, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/de361339e1b84f6db5c770f7c52322d9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/9d8f6147b30e42d4b72498371be55638] to archive 2024-11-17T07:15:00,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:15:00,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/14addfc3568d4a1691424c71c49a6019 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/14addfc3568d4a1691424c71c49a6019 2024-11-17T07:15:00,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5da112d639a841f8aff096330cb3acd6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5da112d639a841f8aff096330cb3acd6 2024-11-17T07:15:00,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d764b5aa4d8048baa37cdb0b1e3cc582 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d764b5aa4d8048baa37cdb0b1e3cc582 2024-11-17T07:15:00,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d1b3725bb25f4ee8ac30575b19f92927 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d1b3725bb25f4ee8ac30575b19f92927 2024-11-17T07:15:00,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/ca1a58b9920d4a62865cd48f4d50635a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/ca1a58b9920d4a62865cd48f4d50635a 2024-11-17T07:15:00,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e18ff74a638c49cabeeef00de606fee4 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e18ff74a638c49cabeeef00de606fee4 2024-11-17T07:15:00,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d486a7186bec4d1e9f0d6cc3bfbf9843 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/d486a7186bec4d1e9f0d6cc3bfbf9843 2024-11-17T07:15:00,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5df402122ebf44e0b477eb8613f1b7f0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5df402122ebf44e0b477eb8613f1b7f0 2024-11-17T07:15:00,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/539c7d6eccde4a15a2b021a0cc7698d5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/539c7d6eccde4a15a2b021a0cc7698d5 2024-11-17T07:15:00,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e176277b329f4ac48009c5f52dd0e2ce to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e176277b329f4ac48009c5f52dd0e2ce 2024-11-17T07:15:00,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/9e2802f856634dcdb85b7afa323262c0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/9e2802f856634dcdb85b7afa323262c0 2024-11-17T07:15:00,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/397c5dda280e4451a38a932faab1742f to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/397c5dda280e4451a38a932faab1742f 2024-11-17T07:15:00,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/2e12dbf5e7364d7e825361288db63bf1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/2e12dbf5e7364d7e825361288db63bf1 2024-11-17T07:15:00,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/6705e0b68ef143afa204fb22b4d8c2d0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/6705e0b68ef143afa204fb22b4d8c2d0 2024-11-17T07:15:00,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/167bc983ca1844b397d3eaa0da2c2a23 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/167bc983ca1844b397d3eaa0da2c2a23 2024-11-17T07:15:00,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/06672b322a424068adf3dbe163d56209 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/06672b322a424068adf3dbe163d56209 2024-11-17T07:15:00,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/781192e106f3466e9e63aa95b4758a7b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/781192e106f3466e9e63aa95b4758a7b 2024-11-17T07:15:00,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/afaa8cf3f3784a8bbbb3c328c7e62981 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/afaa8cf3f3784a8bbbb3c328c7e62981 2024-11-17T07:15:00,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/cbfecdaa1b364d12845b56ad03a42bf1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/cbfecdaa1b364d12845b56ad03a42bf1 2024-11-17T07:15:00,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/01fedcaa494f4c40bcc9423f9f0f9f5c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/01fedcaa494f4c40bcc9423f9f0f9f5c 2024-11-17T07:15:00,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c9c14e50b5294726b24b52a6dd6b6872 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c9c14e50b5294726b24b52a6dd6b6872 2024-11-17T07:15:00,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/56e4c249006b460ba31357155a8beb72 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/56e4c249006b460ba31357155a8beb72 2024-11-17T07:15:00,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/391bac75c9ae44eaac3d7616f6135105 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/391bac75c9ae44eaac3d7616f6135105 2024-11-17T07:15:00,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5a214580b2e94e629f8455c0ae4be26c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/5a214580b2e94e629f8455c0ae4be26c 2024-11-17T07:15:00,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/fef1192605ec4b00867f4a871afdde50 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/fef1192605ec4b00867f4a871afdde50 2024-11-17T07:15:00,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/f6e970870d134fe5ba19f2edbe4b3af7 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/f6e970870d134fe5ba19f2edbe4b3af7 2024-11-17T07:15:00,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c0a7b4f6edd3406b8720b262a3520fd6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c0a7b4f6edd3406b8720b262a3520fd6 2024-11-17T07:15:00,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/41076083de0742a0812dcc21a015ac2c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/41076083de0742a0812dcc21a015ac2c 2024-11-17T07:15:00,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/939d8e9643ae45a2a7fbd898f5d70503 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/939d8e9643ae45a2a7fbd898f5d70503 2024-11-17T07:15:00,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e3ad6add7d0444208707864ca4716240 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e3ad6add7d0444208707864ca4716240 2024-11-17T07:15:00,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/7aa717de61264ea4a397f3a44a4d0dbe to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/7aa717de61264ea4a397f3a44a4d0dbe 2024-11-17T07:15:00,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/de361339e1b84f6db5c770f7c52322d9 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/de361339e1b84f6db5c770f7c52322d9 2024-11-17T07:15:00,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/9d8f6147b30e42d4b72498371be55638 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/9d8f6147b30e42d4b72498371be55638 2024-11-17T07:15:00,822 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/recovered.edits/528.seqid, newMaxSeqId=528, maxSeqId=1 2024-11-17T07:15:00,823 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. 2024-11-17T07:15:00,823 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 6d8c50823ce967d5944d0259cb160d17: Waiting for close lock at 1731827699489Running coprocessor pre-close hooks at 1731827699489Disabling compacts and flushes for region at 1731827699489Disabling writes for close at 1731827699490 (+1 ms)Obtaining lock to block concurrent updates at 1731827699490Preparing flush snapshotting stores in 6d8c50823ce967d5944d0259cb160d17 at 1731827699490Finished memstore snapshotting TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731827699490Flushing stores of TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17. at 1731827699491 (+1 ms)Flushing 6d8c50823ce967d5944d0259cb160d17/A: creating writer at 1731827699491Flushing 6d8c50823ce967d5944d0259cb160d17/A: appending metadata at 1731827699493 (+2 ms)Flushing 6d8c50823ce967d5944d0259cb160d17/A: closing flushed file at 1731827699493Flushing 6d8c50823ce967d5944d0259cb160d17/B: creating writer at 1731827699901 (+408 ms)Flushing 6d8c50823ce967d5944d0259cb160d17/B: appending metadata at 1731827699903 (+2 ms)Flushing 6d8c50823ce967d5944d0259cb160d17/B: closing flushed file at 1731827699903Flushing 6d8c50823ce967d5944d0259cb160d17/C: creating writer at 1731827700310 (+407 ms)Flushing 6d8c50823ce967d5944d0259cb160d17/C: appending metadata at 1731827700312 (+2 ms)Flushing 6d8c50823ce967d5944d0259cb160d17/C: closing flushed file at 1731827700312Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38f764da: reopening flushed file at 1731827700720 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43610da1: reopening flushed file at 1731827700723 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79c11dc4: reopening flushed file at 1731827700727 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6d8c50823ce967d5944d0259cb160d17 in 1241ms, sequenceid=525, compaction requested=true at 1731827700731 (+4 ms)Writing region close event to WAL at 1731827700819 (+88 ms)Running coprocessor post-close hooks at 1731827700823 (+4 ms)Closed at 1731827700823 2024-11-17T07:15:00,824 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:15:00,825 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=6d8c50823ce967d5944d0259cb160d17, regionState=CLOSED 2024-11-17T07:15:00,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:15:00,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-17T07:15:00,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 6d8c50823ce967d5944d0259cb160d17, server=1dd62e24b303,36419,1731827614816 in 1.4880 sec 2024-11-17T07:15:00,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-17T07:15:00,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d8c50823ce967d5944d0259cb160d17, UNASSIGN in 1.4920 sec 2024-11-17T07:15:00,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-17T07:15:00,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4950 sec 2024-11-17T07:15:00,832 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827700832"}]},"ts":"1731827700832"} 2024-11-17T07:15:00,833 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T07:15:00,833 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T07:15:00,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5020 sec 2024-11-17T07:15:01,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-17T07:15:01,477 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:01,478 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T07:15:01,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:01,479 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T07:15:01,480 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:01,482 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:15:01,484 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/recovered.edits] 2024-11-17T07:15:01,486 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4b83bcc96c1a4d608c01dc133fde38db to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/4b83bcc96c1a4d608c01dc133fde38db 2024-11-17T07:15:01,487 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/605c1686f5d14a638dd8e9a117cfe5e1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/605c1686f5d14a638dd8e9a117cfe5e1 2024-11-17T07:15:01,488 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/9ceab27dd1304a7db060ec46e1c1de31 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/A/9ceab27dd1304a7db060ec46e1c1de31 2024-11-17T07:15:01,489 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/1fe44e0025924e80b31ea6860b5b23e9 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/1fe44e0025924e80b31ea6860b5b23e9 2024-11-17T07:15:01,490 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/cc9e421406954c679ed63ffc17ea804c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/cc9e421406954c679ed63ffc17ea804c 2024-11-17T07:15:01,491 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/df2cea28235e4ed9bdbba9596f33c8f6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/B/df2cea28235e4ed9bdbba9596f33c8f6 2024-11-17T07:15:01,493 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c6ecffd652164bb59f1de351099b42bc to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/c6ecffd652164bb59f1de351099b42bc 2024-11-17T07:15:01,493 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e2cb00b35aad4e8386faddb93882bdae to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e2cb00b35aad4e8386faddb93882bdae 2024-11-17T07:15:01,494 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e552d754d01041f1882a377f8b3def86 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/C/e552d754d01041f1882a377f8b3def86 2024-11-17T07:15:01,496 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/recovered.edits/528.seqid to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17/recovered.edits/528.seqid 2024-11-17T07:15:01,497 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/6d8c50823ce967d5944d0259cb160d17 2024-11-17T07:15:01,497 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T07:15:01,498 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:01,501 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T07:15:01,502 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T07:15:01,503 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:01,503 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T07:15:01,504 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731827701503"}]},"ts":"9223372036854775807"} 2024-11-17T07:15:01,505 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T07:15:01,505 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6d8c50823ce967d5944d0259cb160d17, NAME => 'TestAcidGuarantees,,1731827673287.6d8c50823ce967d5944d0259cb160d17.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T07:15:01,505 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T07:15:01,505 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731827701505"}]},"ts":"9223372036854775807"} 2024-11-17T07:15:01,507 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T07:15:01,507 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:01,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 29 msec 2024-11-17T07:15:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-17T07:15:01,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T07:15:01,587 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:01,597 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 252), OpenFileDescriptor=451 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=372 (was 436), ProcessCount=11 (was 11), AvailableMemoryMB=1182 (was 1756) 2024-11-17T07:15:01,605 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=372, ProcessCount=11, AvailableMemoryMB=1182 2024-11-17T07:15:01,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T07:15:01,607 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T07:15:01,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:01,608 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T07:15:01,608 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:01,609 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-17T07:15:01,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-17T07:15:01,609 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T07:15:01,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742190_1366 (size=1061) 2024-11-17T07:15:01,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-17T07:15:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-17T07:15:02,016 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650 2024-11-17T07:15:02,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742191_1367 (size=53) 2024-11-17T07:15:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-17T07:15:02,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:15:02,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d0c00ab4c3958615a6e7b967414841cf, disabling compactions & flushes 2024-11-17T07:15:02,422 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:02,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:02,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. after waiting 0 ms 2024-11-17T07:15:02,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:02,422 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:02,422 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d0c00ab4c3958615a6e7b967414841cf: Waiting for close lock at 1731827702422Disabling compacts and flushes for region at 1731827702422Disabling writes for close at 1731827702422Writing region close event to WAL at 1731827702422Closed at 1731827702422 2024-11-17T07:15:02,423 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T07:15:02,423 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731827702423"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731827702423"}]},"ts":"1731827702423"} 2024-11-17T07:15:02,425 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T07:15:02,426 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T07:15:02,426 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827702426"}]},"ts":"1731827702426"} 2024-11-17T07:15:02,427 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T07:15:02,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, ASSIGN}] 2024-11-17T07:15:02,428 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, ASSIGN 2024-11-17T07:15:02,429 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, ASSIGN; state=OFFLINE, location=1dd62e24b303,36419,1731827614816; forceNewPlan=false, retain=false 2024-11-17T07:15:02,580 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d0c00ab4c3958615a6e7b967414841cf, regionState=OPENING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:02,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, ASSIGN because future has completed 2024-11-17T07:15:02,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:15:02,736 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:02,736 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} 2024-11-17T07:15:02,736 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:02,736 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:15:02,736 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:02,736 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:02,738 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:02,739 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:15:02,739 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c00ab4c3958615a6e7b967414841cf columnFamilyName A 2024-11-17T07:15:02,739 DEBUG [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:02,739 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.HStore(327): Store=d0c00ab4c3958615a6e7b967414841cf/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:15:02,740 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:02,740 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:15:02,741 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c00ab4c3958615a6e7b967414841cf columnFamilyName B 2024-11-17T07:15:02,741 DEBUG [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:02,741 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.HStore(327): Store=d0c00ab4c3958615a6e7b967414841cf/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:15:02,741 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:02,742 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:15:02,742 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c00ab4c3958615a6e7b967414841cf columnFamilyName C 2024-11-17T07:15:02,742 DEBUG [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:02,742 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.HStore(327): Store=d0c00ab4c3958615a6e7b967414841cf/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:15:02,743 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:02,743 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:02,743 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:02,743 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:02,744 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:02,744 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:02,744 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T07:15:02,745 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:02,747 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T07:15:02,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-17T07:15:02,747 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened d0c00ab4c3958615a6e7b967414841cf; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60773316, jitterRate=-0.09440702199935913}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T07:15:02,747 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:02,748 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for d0c00ab4c3958615a6e7b967414841cf: Running coprocessor pre-open hook at 1731827702737Writing region info on filesystem at 1731827702737Initializing all the Stores at 1731827702737Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827702737Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827702737Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827702737Cleaning up temporary data from old regions at 1731827702744 (+7 ms)Running coprocessor post-open hooks at 1731827702747 (+3 ms)Region opened successfully at 1731827702748 (+1 ms) 2024-11-17T07:15:02,749 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., pid=93, masterSystemTime=1731827702733 2024-11-17T07:15:02,750 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:02,750 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:02,751 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d0c00ab4c3958615a6e7b967414841cf, regionState=OPEN, openSeqNum=2, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:02,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:15:02,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-17T07:15:02,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 in 172 msec 2024-11-17T07:15:02,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-17T07:15:02,756 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, ASSIGN in 327 msec 2024-11-17T07:15:02,756 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T07:15:02,756 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827702756"}]},"ts":"1731827702756"} 2024-11-17T07:15:02,758 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T07:15:02,758 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T07:15:02,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-17T07:15:02,931 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T07:15:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-17T07:15:03,758 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:03,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecc882, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:03,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:03,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:03,760 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:03,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:03,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:03,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5524cd5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:03,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:03,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:03,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:03,761 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d159380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:03,762 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:03,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:03,764 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T07:15:03,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 1dd62e24b303,36491,1731827613877 2024-11-17T07:15:03,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3c00b8f8 2024-11-17T07:15:03,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T07:15:03,766 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T07:15:03,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T07:15:03,767 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T07:15:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:03,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742192_1368 (size=1097) 2024-11-17T07:15:04,178 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T07:15:04,178 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T07:15:04,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T07:15:04,182 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, REOPEN/MOVE}] 2024-11-17T07:15:04,183 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, REOPEN/MOVE 2024-11-17T07:15:04,184 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=d0c00ab4c3958615a6e7b967414841cf, regionState=CLOSING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:04,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, REOPEN/MOVE because future has completed 2024-11-17T07:15:04,185 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T07:15:04,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:15:04,337 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,337 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T07:15:04,337 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing d0c00ab4c3958615a6e7b967414841cf, disabling compactions & flushes 2024-11-17T07:15:04,337 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:04,337 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:04,337 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. after waiting 0 ms 2024-11-17T07:15:04,337 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:04,341 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T07:15:04,341 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:04,342 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for d0c00ab4c3958615a6e7b967414841cf: Waiting for close lock at 1731827704337Running coprocessor pre-close hooks at 1731827704337Disabling compacts and flushes for region at 1731827704337Disabling writes for close at 1731827704337Writing region close event to WAL at 1731827704338 (+1 ms)Running coprocessor post-close hooks at 1731827704341 (+3 ms)Closed at 1731827704341 2024-11-17T07:15:04,342 WARN [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: d0c00ab4c3958615a6e7b967414841cf to self. 2024-11-17T07:15:04,343 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,344 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=d0c00ab4c3958615a6e7b967414841cf, regionState=CLOSED 2024-11-17T07:15:04,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:15:04,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-17T07:15:04,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 in 160 msec 2024-11-17T07:15:04,347 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, REOPEN/MOVE; state=CLOSED, location=1dd62e24b303,36419,1731827614816; forceNewPlan=false, retain=true 2024-11-17T07:15:04,387 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T07:15:04,498 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=d0c00ab4c3958615a6e7b967414841cf, regionState=OPENING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:04,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, REOPEN/MOVE because future has completed 2024-11-17T07:15:04,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:15:04,654 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:04,655 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} 2024-11-17T07:15:04,655 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,655 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:15:04,655 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,655 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,656 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,657 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:15:04,657 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c00ab4c3958615a6e7b967414841cf columnFamilyName A 2024-11-17T07:15:04,658 DEBUG [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:04,659 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.HStore(327): Store=d0c00ab4c3958615a6e7b967414841cf/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:15:04,659 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,660 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:15:04,660 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c00ab4c3958615a6e7b967414841cf columnFamilyName B 2024-11-17T07:15:04,660 DEBUG [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:04,660 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.HStore(327): Store=d0c00ab4c3958615a6e7b967414841cf/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:15:04,660 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,661 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:15:04,661 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0c00ab4c3958615a6e7b967414841cf columnFamilyName C 2024-11-17T07:15:04,661 DEBUG [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:04,661 INFO [StoreOpener-d0c00ab4c3958615a6e7b967414841cf-1 {}] regionserver.HStore(327): Store=d0c00ab4c3958615a6e7b967414841cf/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:15:04,661 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:04,662 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,662 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,663 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,663 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,663 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,664 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T07:15:04,665 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,665 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened d0c00ab4c3958615a6e7b967414841cf; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61274909, jitterRate=-0.0869327038526535}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T07:15:04,666 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,666 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for d0c00ab4c3958615a6e7b967414841cf: Running coprocessor pre-open hook at 1731827704655Writing region info on filesystem at 1731827704655Initializing all the Stores at 1731827704656 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827704656Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827704656Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827704656Cleaning up temporary data from old regions at 1731827704663 (+7 ms)Running coprocessor post-open hooks at 1731827704666 (+3 ms)Region opened successfully at 1731827704666 2024-11-17T07:15:04,667 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., pid=98, masterSystemTime=1731827704651 2024-11-17T07:15:04,669 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:04,669 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:04,669 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=d0c00ab4c3958615a6e7b967414841cf, regionState=OPEN, openSeqNum=5, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:04,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:15:04,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-17T07:15:04,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 in 171 msec 2024-11-17T07:15:04,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-17T07:15:04,674 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, REOPEN/MOVE in 490 msec 2024-11-17T07:15:04,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-17T07:15:04,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 494 msec 2024-11-17T07:15:04,677 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 909 msec 2024-11-17T07:15:04,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-17T07:15:04,679 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:04,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f86bdc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,679 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:04,679 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:04,680 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42abade2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,680 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:04,681 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:04,681 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:04,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fed4b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b10b9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,683 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:04,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:04,683 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@622ebdc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:04,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:04,685 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:04,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@593f9b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,686 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c265f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,686 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:04,686 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:04,686 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37611a97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:04,688 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:04,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a5cbe1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,689 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c21b87a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:04,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:04,689 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:04,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:04,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21ff051f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:04,690 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53762, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:04,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@206d7117, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35e136ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:04,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:04,692 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@110cff3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:04,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:04,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:04,693 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:04,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ecd9e81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:04,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:04,695 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:04,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:04,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:04,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d3d8703, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:04,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:04,696 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:04,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5b709, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,697 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35aa2c8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,697 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:04,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:04,698 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ea06a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:04,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:04,699 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:04,699 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73257b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:04,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:04,700 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:04,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f1f880, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:04,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:04,702 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:04,702 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77cd7721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:04,703 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:04,703 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@720ac6b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:04,703 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:04,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:04,704 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:04,705 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f90d551, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:04,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:04,706 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@570c1445, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:04,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:04,707 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:04,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16450616, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:04,708 DEBUG [Thread-1669 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:04,708 DEBUG [Thread-1677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:04,709 DEBUG [Thread-1673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:04,709 DEBUG [Thread-1662 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:04,709 DEBUG [Thread-1664 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:04,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:04,710 DEBUG [Thread-1673 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:04,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:04,710 DEBUG [Thread-1673 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:04,710 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36492, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:04,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:04,711 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:04,711 DEBUG [Thread-1671 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:04,711 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:04,711 DEBUG [Thread-1660 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:04,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:04,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36526, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-17T07:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:15:04,713 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:04,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:04,713 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36550, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:04,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T07:15:04,713 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:15:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:15:04,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:15:04,715 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:04,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:04,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:15:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:15:04,717 DEBUG [Thread-1666 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:04,717 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:15:04,718 DEBUG [Thread-1658 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:04,718 DEBUG [Thread-1675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:04,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:04,719 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:04,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:04,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:15:04,722 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:04,723 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:04,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:15:04,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:04,729 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:04,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:15:04,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:04,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:04,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:04,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:04,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:04,729 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:04,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:15:04,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111723c479d5aff34414ac15fd2f79267de9_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827704721/Put/seqid=0 2024-11-17T07:15:04,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742193_1369 (size=12154) 2024-11-17T07:15:04,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731827704743, max timestamp=1731827704766Num uniques 859; 2024-11-17T07:15:04,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731827704743, max timestamp=1731827704766Num uniques 859; 2024-11-17T07:15:04,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T07:15:04,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T07:15:04,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731827704743, max timestamp=1731827704766Num uniques 859; 2024-11-17T07:15:04,770 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T07:15:04,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:04,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827764786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:04,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827764788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:04,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827764790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:04,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827764798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:04,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:04,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827764800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:04,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T07:15:04,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T07:15:04,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:04,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:04,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:04,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:04,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:04,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:04,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:04,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827764907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:04,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827764907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:04,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827764907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:04,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827764907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:04,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:04,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827764917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:04,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:04,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T07:15:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:05,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T07:15:05,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827765127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827765127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827765127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827765128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827765137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,166 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:05,170 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111723c479d5aff34414ac15fd2f79267de9_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111723c479d5aff34414ac15fd2f79267de9_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:05,171 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/444d687d4452408e87f6b1a309c4d3fd, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:05,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/444d687d4452408e87f6b1a309c4d3fd is 175, key is test_row_0/A:col10/1731827704721/Put/seqid=0 2024-11-17T07:15:05,175 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T07:15:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:05,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742194_1370 (size=30955) 2024-11-17T07:15:05,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,178 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/444d687d4452408e87f6b1a309c4d3fd 2024-11-17T07:15:05,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/c8b6b9f6b74e4e43b504812dd2514fb0 is 50, key is test_row_0/B:col10/1731827704721/Put/seqid=0 2024-11-17T07:15:05,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742195_1371 (size=12001) 2024-11-17T07:15:05,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/c8b6b9f6b74e4e43b504812dd2514fb0 2024-11-17T07:15:05,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/44eb0517d2e341a7bce7188774c1aa34 is 50, key is test_row_0/C:col10/1731827704721/Put/seqid=0 2024-11-17T07:15:05,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742196_1372 (size=12001) 2024-11-17T07:15:05,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T07:15:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:05,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T07:15:05,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827765447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827765447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827765447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827765448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,458 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827765448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T07:15:05,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:05,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:05,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:05,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/44eb0517d2e341a7bce7188774c1aa34 2024-11-17T07:15:05,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T07:15:05,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:05,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:05,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:05,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/444d687d4452408e87f6b1a309c4d3fd as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/444d687d4452408e87f6b1a309c4d3fd 2024-11-17T07:15:05,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:05,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/444d687d4452408e87f6b1a309c4d3fd, entries=150, sequenceid=14, filesize=30.2 K 2024-11-17T07:15:05,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/c8b6b9f6b74e4e43b504812dd2514fb0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/c8b6b9f6b74e4e43b504812dd2514fb0 2024-11-17T07:15:05,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/c8b6b9f6b74e4e43b504812dd2514fb0, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T07:15:05,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/44eb0517d2e341a7bce7188774c1aa34 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/44eb0517d2e341a7bce7188774c1aa34 2024-11-17T07:15:05,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/44eb0517d2e341a7bce7188774c1aa34, entries=150, sequenceid=14, filesize=11.7 K 2024-11-17T07:15:05,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for d0c00ab4c3958615a6e7b967414841cf in 922ms, sequenceid=14, compaction requested=false 2024-11-17T07:15:05,651 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T07:15:05,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:05,788 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-17T07:15:05,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:05,788 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-17T07:15:05,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:05,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:05,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:05,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:05,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:05,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:05,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174c9b2b73ea1042c6bbbcc47ec9ba3fa6_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827704797/Put/seqid=0 2024-11-17T07:15:05,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742197_1373 (size=12154) 2024-11-17T07:15:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T07:15:05,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:05,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:05,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827765976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827765977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827765978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827765981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:05,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:05,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827765990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:05,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827766097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827766098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827766098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827766098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827766098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:06,205 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174c9b2b73ea1042c6bbbcc47ec9ba3fa6_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174c9b2b73ea1042c6bbbcc47ec9ba3fa6_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:06,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/b99a22160061444cb18300fb048db293, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:06,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/b99a22160061444cb18300fb048db293 is 175, key is test_row_0/A:col10/1731827704797/Put/seqid=0 2024-11-17T07:15:06,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742198_1374 (size=30955) 2024-11-17T07:15:06,233 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/b99a22160061444cb18300fb048db293 2024-11-17T07:15:06,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/7f800554ca7c43af81956222075077b0 is 50, key is test_row_0/B:col10/1731827704797/Put/seqid=0 2024-11-17T07:15:06,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742199_1375 (size=12001) 2024-11-17T07:15:06,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827766318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827766318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827766318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827766318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827766318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,348 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T07:15:06,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827766638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827766638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827766638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827766638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827766638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:06,683 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/7f800554ca7c43af81956222075077b0 2024-11-17T07:15:06,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/e2798f53ea5049ff812a4b15cd4635ab is 50, key is test_row_0/C:col10/1731827704797/Put/seqid=0 2024-11-17T07:15:06,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742200_1376 (size=12001) 2024-11-17T07:15:06,696 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/e2798f53ea5049ff812a4b15cd4635ab 2024-11-17T07:15:06,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/b99a22160061444cb18300fb048db293 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/b99a22160061444cb18300fb048db293 2024-11-17T07:15:06,708 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/b99a22160061444cb18300fb048db293, entries=150, sequenceid=48, filesize=30.2 K 2024-11-17T07:15:06,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/7f800554ca7c43af81956222075077b0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7f800554ca7c43af81956222075077b0 2024-11-17T07:15:06,713 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7f800554ca7c43af81956222075077b0, entries=150, sequenceid=48, filesize=11.7 K 2024-11-17T07:15:06,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/e2798f53ea5049ff812a4b15cd4635ab as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/e2798f53ea5049ff812a4b15cd4635ab 2024-11-17T07:15:06,718 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/e2798f53ea5049ff812a4b15cd4635ab, entries=150, sequenceid=48, filesize=11.7 K 2024-11-17T07:15:06,719 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for d0c00ab4c3958615a6e7b967414841cf in 931ms, sequenceid=48, compaction requested=false 2024-11-17T07:15:06,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:06,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:06,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-17T07:15:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-17T07:15:06,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-17T07:15:06,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0060 sec 2024-11-17T07:15:06,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.0130 sec 2024-11-17T07:15:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-17T07:15:06,857 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:06,857 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-17T07:15:06,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T07:15:06,860 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:06,860 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:06,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:06,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T07:15:07,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-17T07:15:07,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:07,014 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:15:07,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:07,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:07,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:07,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:07,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:07,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:07,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178316a29a2fb847a982b099c8421c1211_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827705979/Put/seqid=0 2024-11-17T07:15:07,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742201_1377 (size=12154) 2024-11-17T07:15:07,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:07,051 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411178316a29a2fb847a982b099c8421c1211_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178316a29a2fb847a982b099c8421c1211_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:07,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/742385263d7345f5858cba608b7d4b17, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:07,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/742385263d7345f5858cba608b7d4b17 is 175, key is test_row_0/A:col10/1731827705979/Put/seqid=0 2024-11-17T07:15:07,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742202_1378 (size=30955) 2024-11-17T07:15:07,071 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=58, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/742385263d7345f5858cba608b7d4b17 2024-11-17T07:15:07,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/da1b4f153e2a41debe9cc57ccd293abe is 50, key is test_row_0/B:col10/1731827705979/Put/seqid=0 2024-11-17T07:15:07,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742203_1379 (size=12001) 2024-11-17T07:15:07,095 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/da1b4f153e2a41debe9cc57ccd293abe 2024-11-17T07:15:07,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/a7a49c5ca33c4060ae30e82215ba7503 is 50, key is test_row_0/C:col10/1731827705979/Put/seqid=0 2024-11-17T07:15:07,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742204_1380 (size=12001) 2024-11-17T07:15:07,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:07,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T07:15:07,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827707176, max timestamp=1731827707196Num uniques 859; 2024-11-17T07:15:07,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827707176, max timestamp=1731827707196Num uniques 859; 2024-11-17T07:15:07,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:15:07,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:15:07,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827707176, max timestamp=1731827707196Num uniques 859; 2024-11-17T07:15:07,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:15:07,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827767236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,240 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827767237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827767237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827767238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,252 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827767239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,253 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827767347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827767358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827767358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827767358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827767358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T07:15:07,515 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/a7a49c5ca33c4060ae30e82215ba7503 2024-11-17T07:15:07,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/742385263d7345f5858cba608b7d4b17 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/742385263d7345f5858cba608b7d4b17 2024-11-17T07:15:07,527 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/742385263d7345f5858cba608b7d4b17, entries=150, sequenceid=58, filesize=30.2 K 2024-11-17T07:15:07,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/da1b4f153e2a41debe9cc57ccd293abe as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/da1b4f153e2a41debe9cc57ccd293abe 2024-11-17T07:15:07,536 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/da1b4f153e2a41debe9cc57ccd293abe, entries=150, sequenceid=58, filesize=11.7 K 2024-11-17T07:15:07,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/a7a49c5ca33c4060ae30e82215ba7503 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/a7a49c5ca33c4060ae30e82215ba7503 2024-11-17T07:15:07,541 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/a7a49c5ca33c4060ae30e82215ba7503, entries=150, sequenceid=58, filesize=11.7 K 2024-11-17T07:15:07,542 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for d0c00ab4c3958615a6e7b967414841cf in 527ms, sequenceid=58, compaction requested=true 2024-11-17T07:15:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:07,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-17T07:15:07,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-17T07:15:07,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-17T07:15:07,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 683 msec 2024-11-17T07:15:07,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 691 msec 2024-11-17T07:15:07,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:07,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-17T07:15:07,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:07,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:07,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:07,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:07,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:07,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:07,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170dabe548ca8d4a78bbe69c0fa81a25b1_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827707236/Put/seqid=0 2024-11-17T07:15:07,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827767587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827767591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827767593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,609 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827767595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742205_1381 (size=14594) 2024-11-17T07:15:07,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827767602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827767717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827767717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827767717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,727 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827767717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827767907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,921 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827767937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827767937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827767937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,943 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827767937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:07,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:07,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-17T07:15:07,997 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:07,997 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:07,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-17T07:15:07,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T07:15:07,999 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:08,000 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:08,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:08,013 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:08,017 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170dabe548ca8d4a78bbe69c0fa81a25b1_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170dabe548ca8d4a78bbe69c0fa81a25b1_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:08,017 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/ed4c0b38642043899b1875e286f32938, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:08,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/ed4c0b38642043899b1875e286f32938 is 175, key is test_row_0/A:col10/1731827707236/Put/seqid=0 2024-11-17T07:15:08,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742206_1382 (size=39549) 2024-11-17T07:15:08,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T07:15:08,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T07:15:08,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:08,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:08,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:08,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827768247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:08,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827768247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:08,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827768247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:08,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827768247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:08,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T07:15:08,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:08,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:08,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T07:15:08,424 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/ed4c0b38642043899b1875e286f32938 2024-11-17T07:15:08,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/194c5c922655462ebf66bdffc000ee71 is 50, key is test_row_0/B:col10/1731827707236/Put/seqid=0 2024-11-17T07:15:08,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742207_1383 (size=12001) 2024-11-17T07:15:08,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:08,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827768427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,438 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:08,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T07:15:08,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:08,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:08,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:08,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T07:15:08,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:08,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:08,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:08,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T07:15:08,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T07:15:08,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:08,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:08,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:08,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827768767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:08,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827768768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:08,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827768768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,778 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:08,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:08,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827768768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:08,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/194c5c922655462ebf66bdffc000ee71 2024-11-17T07:15:08,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/3b50e12eb0534ffab65b2ed1a63ce2d9 is 50, key is test_row_0/C:col10/1731827707236/Put/seqid=0 2024-11-17T07:15:08,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742208_1384 (size=12001) 2024-11-17T07:15:08,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T07:15:08,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:08,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:08,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:08,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:08,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:09,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T07:15:09,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:09,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:09,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:09,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:09,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:09,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:09,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T07:15:09,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T07:15:09,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:09,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:09,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:09,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:09,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:09,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/3b50e12eb0534ffab65b2ed1a63ce2d9 2024-11-17T07:15:09,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/ed4c0b38642043899b1875e286f32938 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/ed4c0b38642043899b1875e286f32938 2024-11-17T07:15:09,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/ed4c0b38642043899b1875e286f32938, entries=200, sequenceid=95, filesize=38.6 K 2024-11-17T07:15:09,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/194c5c922655462ebf66bdffc000ee71 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/194c5c922655462ebf66bdffc000ee71 2024-11-17T07:15:09,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/194c5c922655462ebf66bdffc000ee71, entries=150, sequenceid=95, filesize=11.7 K 2024-11-17T07:15:09,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/3b50e12eb0534ffab65b2ed1a63ce2d9 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3b50e12eb0534ffab65b2ed1a63ce2d9 2024-11-17T07:15:09,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3b50e12eb0534ffab65b2ed1a63ce2d9, entries=150, sequenceid=95, filesize=11.7 K 2024-11-17T07:15:09,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for d0c00ab4c3958615a6e7b967414841cf in 1690ms, sequenceid=95, compaction requested=true 2024-11-17T07:15:09,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:09,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:09,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:09,270 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:15:09,270 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:15:09,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:09,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:09,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:09,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:09,271 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:15:09,271 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/B is initiating minor compaction (all files) 2024-11-17T07:15:09,271 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/B in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:09,272 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/c8b6b9f6b74e4e43b504812dd2514fb0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7f800554ca7c43af81956222075077b0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/da1b4f153e2a41debe9cc57ccd293abe, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/194c5c922655462ebf66bdffc000ee71] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=46.9 K 2024-11-17T07:15:09,272 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:15:09,272 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/A is initiating minor compaction (all files) 2024-11-17T07:15:09,272 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/A in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:09,272 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/444d687d4452408e87f6b1a309c4d3fd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/b99a22160061444cb18300fb048db293, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/742385263d7345f5858cba608b7d4b17, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/ed4c0b38642043899b1875e286f32938] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=129.3 K 2024-11-17T07:15:09,272 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:09,272 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/444d687d4452408e87f6b1a309c4d3fd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/b99a22160061444cb18300fb048db293, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/742385263d7345f5858cba608b7d4b17, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/ed4c0b38642043899b1875e286f32938] 2024-11-17T07:15:09,272 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting c8b6b9f6b74e4e43b504812dd2514fb0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731827704721 2024-11-17T07:15:09,272 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 444d687d4452408e87f6b1a309c4d3fd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731827704721 2024-11-17T07:15:09,273 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f800554ca7c43af81956222075077b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731827704788 2024-11-17T07:15:09,273 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting b99a22160061444cb18300fb048db293, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731827704788 2024-11-17T07:15:09,273 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting da1b4f153e2a41debe9cc57ccd293abe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731827705974 2024-11-17T07:15:09,273 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 742385263d7345f5858cba608b7d4b17, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731827705974 2024-11-17T07:15:09,274 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed4c0b38642043899b1875e286f32938, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731827707221 2024-11-17T07:15:09,274 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 194c5c922655462ebf66bdffc000ee71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731827707236 2024-11-17T07:15:09,293 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:09,295 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#B#compaction#330 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:09,296 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/a3dc7fb42b3e42e5b9cb84a5438213c0 is 50, key is test_row_0/B:col10/1731827707236/Put/seqid=0 2024-11-17T07:15:09,298 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117d46104b41af0438284adf88eb3e78bd0_d0c00ab4c3958615a6e7b967414841cf store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:09,300 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117d46104b41af0438284adf88eb3e78bd0_d0c00ab4c3958615a6e7b967414841cf, store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:09,301 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d46104b41af0438284adf88eb3e78bd0_d0c00ab4c3958615a6e7b967414841cf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:09,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742209_1385 (size=12139) 2024-11-17T07:15:09,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742210_1386 (size=4469) 2024-11-17T07:15:09,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-17T07:15:09,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:09,378 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-17T07:15:09,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:09,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ff728ed0da724cfe9fe630abf3f98dbc_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827707600/Put/seqid=0 2024-11-17T07:15:09,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742211_1387 (size=12154) 2024-11-17T07:15:09,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:09,410 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ff728ed0da724cfe9fe630abf3f98dbc_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ff728ed0da724cfe9fe630abf3f98dbc_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/5555ee63799a46b0a54607f9ac8d1df6, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:09,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/5555ee63799a46b0a54607f9ac8d1df6 is 175, key is test_row_0/A:col10/1731827707600/Put/seqid=0 2024-11-17T07:15:09,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742212_1388 (size=30955) 2024-11-17T07:15:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:09,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:09,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827709577, max timestamp=1731827709621Num uniques 859; 2024-11-17T07:15:09,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827709577, max timestamp=1731827709621Num uniques 859; 2024-11-17T07:15:09,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827709577, max timestamp=1731827709621Num uniques 859; 2024-11-17T07:15:09,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:09,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:09,624 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:09,710 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/a3dc7fb42b3e42e5b9cb84a5438213c0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a3dc7fb42b3e42e5b9cb84a5438213c0 2024-11-17T07:15:09,713 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#A#compaction#331 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:09,713 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/1da8549fe0d54ca0b87be9dcdbe023af is 175, key is test_row_0/A:col10/1731827707236/Put/seqid=0 2024-11-17T07:15:09,714 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/B of d0c00ab4c3958615a6e7b967414841cf into a3dc7fb42b3e42e5b9cb84a5438213c0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:09,714 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:09,714 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/B, priority=12, startTime=1731827709270; duration=0sec 2024-11-17T07:15:09,714 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:09,715 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:B 2024-11-17T07:15:09,715 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:15:09,716 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:15:09,716 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/C is initiating minor compaction (all files) 2024-11-17T07:15:09,716 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/C in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:09,716 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/44eb0517d2e341a7bce7188774c1aa34, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/e2798f53ea5049ff812a4b15cd4635ab, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/a7a49c5ca33c4060ae30e82215ba7503, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3b50e12eb0534ffab65b2ed1a63ce2d9] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=46.9 K 2024-11-17T07:15:09,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742213_1389 (size=31093) 2024-11-17T07:15:09,717 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 44eb0517d2e341a7bce7188774c1aa34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731827704721 2024-11-17T07:15:09,717 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting e2798f53ea5049ff812a4b15cd4635ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731827704788 2024-11-17T07:15:09,718 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a7a49c5ca33c4060ae30e82215ba7503, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731827705974 2024-11-17T07:15:09,718 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b50e12eb0534ffab65b2ed1a63ce2d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731827707236 2024-11-17T07:15:09,722 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/1da8549fe0d54ca0b87be9dcdbe023af as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/1da8549fe0d54ca0b87be9dcdbe023af 2024-11-17T07:15:09,727 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/A of d0c00ab4c3958615a6e7b967414841cf into 1da8549fe0d54ca0b87be9dcdbe023af(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:09,727 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:09,727 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/A, priority=12, startTime=1731827709270; duration=0sec 2024-11-17T07:15:09,727 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:09,727 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:A 2024-11-17T07:15:09,728 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#C#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:09,729 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/59155d3b650c4850bcd481a791055f2d is 50, key is test_row_0/C:col10/1731827707236/Put/seqid=0 2024-11-17T07:15:09,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742214_1390 (size=12139) 2024-11-17T07:15:09,737 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/59155d3b650c4850bcd481a791055f2d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/59155d3b650c4850bcd481a791055f2d 2024-11-17T07:15:09,745 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/C of d0c00ab4c3958615a6e7b967414841cf into 59155d3b650c4850bcd481a791055f2d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:09,745 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:09,745 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/C, priority=12, startTime=1731827709270; duration=0sec 2024-11-17T07:15:09,745 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:09,745 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:C 2024-11-17T07:15:09,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827769787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:09,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827769787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:09,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827769787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:09,804 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in d0c00ab4c3958615a6e7b967414841cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:09,804 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in d0c00ab4c3958615a6e7b967414841cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:09,805 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:09,805 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in d0c00ab4c3958615a6e7b967414841cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:09,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:09,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731827709718, max timestamp=1731827709802Num uniques 1018; 2024-11-17T07:15:09,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T07:15:09,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731827709718, max timestamp=1731827709802Num uniques 1018; 2024-11-17T07:15:09,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T07:15:09,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731827709718, max timestamp=1731827709802Num uniques 1018; 2024-11-17T07:15:09,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-17T07:15:09,830 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=104, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/5555ee63799a46b0a54607f9ac8d1df6 2024-11-17T07:15:09,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/049c09ac3a4040a68634c12ca6416e5d is 50, key is test_row_0/B:col10/1731827707600/Put/seqid=0 2024-11-17T07:15:09,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742215_1391 (size=12001) 2024-11-17T07:15:09,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827769899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:09,914 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:09,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731827709878, max timestamp=1731827709911Num uniques 1027; 2024-11-17T07:15:09,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-17T07:15:09,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731827709878, max timestamp=1731827709911Num uniques 1027; 2024-11-17T07:15:09,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-17T07:15:09,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731827709878, max timestamp=1731827709911Num uniques 1027; 2024-11-17T07:15:09,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-17T07:15:10,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731827709998, max timestamp=1731827710038Num uniques 1036; 2024-11-17T07:15:10,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T07:15:10,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731827709998, max timestamp=1731827710038Num uniques 1036; 2024-11-17T07:15:10,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T07:15:10,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731827709998, max timestamp=1731827710038Num uniques 1036; 2024-11-17T07:15:10,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-17T07:15:10,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731827710054, max timestamp=1731827710126Num uniques 1045; 2024-11-17T07:15:10,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731827710054, max timestamp=1731827710126Num uniques 1045; 2024-11-17T07:15:10,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-17T07:15:10,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731827710054, max timestamp=1731827710126Num uniques 1045; 2024-11-17T07:15:10,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-17T07:15:10,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-17T07:15:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T07:15:10,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827770201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:10,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827770203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:10,242 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/049c09ac3a4040a68634c12ca6416e5d 2024-11-17T07:15:10,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/428935610ded49359ff9f2a05613a459 is 50, key is test_row_0/C:col10/1731827707600/Put/seqid=0 2024-11-17T07:15:10,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742216_1392 (size=12001) 2024-11-17T07:15:10,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827770317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:10,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827770317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,329 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:10,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:10,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827770537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:10,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827770537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:10,654 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/428935610ded49359ff9f2a05613a459 2024-11-17T07:15:10,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/5555ee63799a46b0a54607f9ac8d1df6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5555ee63799a46b0a54607f9ac8d1df6 2024-11-17T07:15:10,662 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5555ee63799a46b0a54607f9ac8d1df6, entries=150, sequenceid=104, filesize=30.2 K 2024-11-17T07:15:10,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/049c09ac3a4040a68634c12ca6416e5d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/049c09ac3a4040a68634c12ca6416e5d 2024-11-17T07:15:10,666 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/049c09ac3a4040a68634c12ca6416e5d, entries=150, sequenceid=104, filesize=11.7 K 2024-11-17T07:15:10,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/428935610ded49359ff9f2a05613a459 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/428935610ded49359ff9f2a05613a459 2024-11-17T07:15:10,670 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/428935610ded49359ff9f2a05613a459, entries=150, sequenceid=104, filesize=11.7 K 2024-11-17T07:15:10,671 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for d0c00ab4c3958615a6e7b967414841cf in 1292ms, sequenceid=104, compaction requested=false 2024-11-17T07:15:10,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:10,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:10,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-17T07:15:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-17T07:15:10,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-17T07:15:10,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6720 sec 2024-11-17T07:15:10,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.6770 sec 2024-11-17T07:15:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:10,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=147.60 KB heapSize=416.77 KB 2024-11-17T07:15:10,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:10,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:10,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:10,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:10,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:10,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:10,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174243b3ddc29c4da7852d32d4834f94f6_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827710862/Put/seqid=0 2024-11-17T07:15:10,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742217_1393 (size=14794) 2024-11-17T07:15:10,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:10,880 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174243b3ddc29c4da7852d32d4834f94f6_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174243b3ddc29c4da7852d32d4834f94f6_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:10,881 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/82d714f654d4478d9619c504884fd8eb, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:10,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/82d714f654d4478d9619c504884fd8eb is 175, key is test_row_0/A:col10/1731827710862/Put/seqid=0 2024-11-17T07:15:10,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742218_1394 (size=39749) 2024-11-17T07:15:10,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827770880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:10,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,891 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827770885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:10,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:10,892 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827770997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:11,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827770997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:11,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:11,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:11,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827771207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,216 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:11,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:11,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827771217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:11,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,230 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:11,287 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/82d714f654d4478d9619c504884fd8eb 2024-11-17T07:15:11,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/a2fb45b9c1164c03923811f3612d461c is 50, key is test_row_0/B:col10/1731827710862/Put/seqid=0 2024-11-17T07:15:11,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742219_1395 (size=12151) 2024-11-17T07:15:11,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:11,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827771527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:11,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:11,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827771537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:11,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/a2fb45b9c1164c03923811f3612d461c 2024-11-17T07:15:11,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/a29fcdb977544815b1d129b1362713d0 is 50, key is test_row_0/C:col10/1731827710862/Put/seqid=0 2024-11-17T07:15:11,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742220_1396 (size=12151) 2024-11-17T07:15:11,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/a29fcdb977544815b1d129b1362713d0 2024-11-17T07:15:11,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/82d714f654d4478d9619c504884fd8eb as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/82d714f654d4478d9619c504884fd8eb 2024-11-17T07:15:11,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/82d714f654d4478d9619c504884fd8eb, entries=200, sequenceid=213, filesize=38.8 K 2024-11-17T07:15:11,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/a2fb45b9c1164c03923811f3612d461c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a2fb45b9c1164c03923811f3612d461c 2024-11-17T07:15:11,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a2fb45b9c1164c03923811f3612d461c, entries=150, sequenceid=213, filesize=11.9 K 2024-11-17T07:15:11,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/a29fcdb977544815b1d129b1362713d0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/a29fcdb977544815b1d129b1362713d0 2024-11-17T07:15:11,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/a29fcdb977544815b1d129b1362713d0, entries=150, sequenceid=213, filesize=11.9 K 2024-11-17T07:15:11,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~416.72 KB/426720, currentSize=33.54 KB/34350 for d0c00ab4c3958615a6e7b967414841cf in 863ms, sequenceid=213, compaction requested=true 2024-11-17T07:15:11,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:11,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:11,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:11,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:11,726 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:11,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:11,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:11,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-17T07:15:11,726 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:11,727 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:11,727 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:11,727 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/A is initiating minor compaction (all files) 2024-11-17T07:15:11,727 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/B is initiating minor compaction (all files) 2024-11-17T07:15:11,727 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/B in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:11,727 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/A in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:11,727 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a3dc7fb42b3e42e5b9cb84a5438213c0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/049c09ac3a4040a68634c12ca6416e5d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a2fb45b9c1164c03923811f3612d461c] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=35.4 K 2024-11-17T07:15:11,727 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/1da8549fe0d54ca0b87be9dcdbe023af, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5555ee63799a46b0a54607f9ac8d1df6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/82d714f654d4478d9619c504884fd8eb] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=99.4 K 2024-11-17T07:15:11,727 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:11,727 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/1da8549fe0d54ca0b87be9dcdbe023af, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5555ee63799a46b0a54607f9ac8d1df6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/82d714f654d4478d9619c504884fd8eb] 2024-11-17T07:15:11,727 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a3dc7fb42b3e42e5b9cb84a5438213c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731827707236 2024-11-17T07:15:11,728 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1da8549fe0d54ca0b87be9dcdbe023af, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731827707236 2024-11-17T07:15:11,728 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 049c09ac3a4040a68634c12ca6416e5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731827707591 2024-11-17T07:15:11,728 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5555ee63799a46b0a54607f9ac8d1df6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731827707591 2024-11-17T07:15:11,728 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a2fb45b9c1164c03923811f3612d461c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731827710171 2024-11-17T07:15:11,728 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82d714f654d4478d9619c504884fd8eb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731827710171 2024-11-17T07:15:11,737 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:11,738 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#B#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:11,738 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/f8a55aa1935146fd8e5ccb917a9cb713 is 50, key is test_row_0/B:col10/1731827710862/Put/seqid=0 2024-11-17T07:15:11,739 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411170b82b4cf3cd24f6fbd749afb74e9f6a6_d0c00ab4c3958615a6e7b967414841cf store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:11,741 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411170b82b4cf3cd24f6fbd749afb74e9f6a6_d0c00ab4c3958615a6e7b967414841cf, store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:11,742 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170b82b4cf3cd24f6fbd749afb74e9f6a6_d0c00ab4c3958615a6e7b967414841cf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:11,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742221_1397 (size=12391) 2024-11-17T07:15:11,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742222_1398 (size=4469) 2024-11-17T07:15:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:11,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:15:11,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:11,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:11,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:11,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:11,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:11,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:11,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170849f50adc3846339b562bf394962d2c_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827711833/Put/seqid=0 2024-11-17T07:15:11,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742223_1399 (size=19774) 2024-11-17T07:15:11,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827711873, max timestamp=1731827711893Num uniques 859; 2024-11-17T07:15:11,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:15:11,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827711873, max timestamp=1731827711893Num uniques 859; 2024-11-17T07:15:11,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:15:11,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827711873, max timestamp=1731827711893Num uniques 859; 2024-11-17T07:15:11,897 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:15:11,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:11,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827771944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,956 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:11,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827771945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:11,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827771952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:11,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:12,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:12,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827772047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:12,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:12,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:12,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827772058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:12,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:12,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:12,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827772067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:12,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:12,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827772067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:12,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,079 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:12,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:12,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827772067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:12,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:12,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:12,148 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#A#compaction#340 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:12,149 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/43047641aadb4f73b0ef3459bd4bdbec is 175, key is test_row_0/A:col10/1731827710862/Put/seqid=0 2024-11-17T07:15:12,150 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/f8a55aa1935146fd8e5ccb917a9cb713 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/f8a55aa1935146fd8e5ccb917a9cb713 2024-11-17T07:15:12,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742224_1400 (size=31345) 2024-11-17T07:15:12,155 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/B of d0c00ab4c3958615a6e7b967414841cf into f8a55aa1935146fd8e5ccb917a9cb713(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:12,155 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:12,155 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/B, priority=13, startTime=1731827711726; duration=0sec 2024-11-17T07:15:12,155 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:12,155 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:B 2024-11-17T07:15:12,155 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:12,156 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:12,156 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/C is initiating minor compaction (all files) 2024-11-17T07:15:12,156 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/C in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:12,156 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/59155d3b650c4850bcd481a791055f2d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/428935610ded49359ff9f2a05613a459, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/a29fcdb977544815b1d129b1362713d0] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=35.4 K 2024-11-17T07:15:12,157 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 59155d3b650c4850bcd481a791055f2d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731827707236 2024-11-17T07:15:12,157 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 428935610ded49359ff9f2a05613a459, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1731827707591 2024-11-17T07:15:12,157 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a29fcdb977544815b1d129b1362713d0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731827710171 2024-11-17T07:15:12,163 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:12,163 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/062e459681234604aac8c859169f2808 is 50, key is test_row_0/C:col10/1731827710862/Put/seqid=0 2024-11-17T07:15:12,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-17T07:15:12,167 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:12,168 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:12,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-17T07:15:12,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T07:15:12,170 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:12,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742225_1401 (size=12391) 2024-11-17T07:15:12,171 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:12,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:12,250 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:12,254 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170849f50adc3846339b562bf394962d2c_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170849f50adc3846339b562bf394962d2c_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:12,255 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/da0fa3b02fde4528b13cfa0f57f40dd5, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:12,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/da0fa3b02fde4528b13cfa0f57f40dd5 is 175, key is test_row_0/A:col10/1731827711833/Put/seqid=0 2024-11-17T07:15:12,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742226_1402 (size=57033) 2024-11-17T07:15:12,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T07:15:12,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827772287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827772287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:12,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827772287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:12,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T07:15:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:12,323 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T07:15:12,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:12,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:12,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:12,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T07:15:12,557 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/43047641aadb4f73b0ef3459bd4bdbec as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/43047641aadb4f73b0ef3459bd4bdbec 2024-11-17T07:15:12,561 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/A of d0c00ab4c3958615a6e7b967414841cf into 43047641aadb4f73b0ef3459bd4bdbec(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:12,562 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:12,562 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/A, priority=13, startTime=1731827711726; duration=0sec 2024-11-17T07:15:12,562 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:12,562 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:A 2024-11-17T07:15:12,575 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/062e459681234604aac8c859169f2808 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/062e459681234604aac8c859169f2808 2024-11-17T07:15:12,579 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/C of d0c00ab4c3958615a6e7b967414841cf into 062e459681234604aac8c859169f2808(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:12,579 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:12,579 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/C, priority=13, startTime=1731827711726; duration=0sec 2024-11-17T07:15:12,579 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:12,579 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:C 2024-11-17T07:15:12,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827772607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827772607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:12,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827772607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:12,619 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:12,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T07:15:12,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:12,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:12,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:12,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,659 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/da0fa3b02fde4528b13cfa0f57f40dd5 2024-11-17T07:15:12,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/f19ddb9e87f14f17b710757ae2b38b54 is 50, key is test_row_0/B:col10/1731827711833/Put/seqid=0 2024-11-17T07:15:12,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742227_1403 (size=12151) 2024-11-17T07:15:12,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T07:15:12,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:12,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T07:15:12,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T07:15:12,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:12,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:12,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:12,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:12,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:13,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/f19ddb9e87f14f17b710757ae2b38b54 2024-11-17T07:15:13,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827773067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:13,079 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in d0c00ab4c3958615a6e7b967414841cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:13,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:13,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/5ba39c79161a4915b87af676c2dcba11 is 50, key is test_row_0/C:col10/1731827711833/Put/seqid=0 2024-11-17T07:15:13,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742228_1404 (size=12151) 2024-11-17T07:15:13,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:13,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827773077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:13,088 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in d0c00ab4c3958615a6e7b967414841cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:13,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T07:15:13,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:13,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:13,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:13,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:13,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:13,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827773127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:13,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:13,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827773127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:13,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:13,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:13,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827773127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:13,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:13,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T07:15:13,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:13,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:13,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:13,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:13,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:13,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T07:15:13,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T07:15:13,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:13,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:13,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:13,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:13,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:13,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/5ba39c79161a4915b87af676c2dcba11 2024-11-17T07:15:13,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/da0fa3b02fde4528b13cfa0f57f40dd5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/da0fa3b02fde4528b13cfa0f57f40dd5 2024-11-17T07:15:13,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/da0fa3b02fde4528b13cfa0f57f40dd5, entries=300, sequenceid=224, filesize=55.7 K 2024-11-17T07:15:13,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/f19ddb9e87f14f17b710757ae2b38b54 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/f19ddb9e87f14f17b710757ae2b38b54 2024-11-17T07:15:13,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/f19ddb9e87f14f17b710757ae2b38b54, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T07:15:13,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/5ba39c79161a4915b87af676c2dcba11 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5ba39c79161a4915b87af676c2dcba11 2024-11-17T07:15:13,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5ba39c79161a4915b87af676c2dcba11, entries=150, sequenceid=224, filesize=11.9 K 2024-11-17T07:15:13,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for d0c00ab4c3958615a6e7b967414841cf in 1670ms, sequenceid=224, compaction requested=false 2024-11-17T07:15:13,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:13,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-17T07:15:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:13,548 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-17T07:15:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:13,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:13,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:13,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:13,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:13,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117934a55a7ee014dc69b6c8d3b4db8cec2_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827711925/Put/seqid=0 2024-11-17T07:15:13,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742229_1405 (size=12354) 2024-11-17T07:15:13,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:13,967 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117934a55a7ee014dc69b6c8d3b4db8cec2_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117934a55a7ee014dc69b6c8d3b4db8cec2_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:13,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/1361bde36c424f889f39ae2966fe36f3, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:13,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/1361bde36c424f889f39ae2966fe36f3 is 175, key is test_row_0/A:col10/1731827711925/Put/seqid=0 2024-11-17T07:15:13,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742230_1406 (size=31155) 2024-11-17T07:15:14,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:14,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:14,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827774173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,180 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:14,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:14,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827774178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:14,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827774179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:14,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:14,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827774287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827774297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:14,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827774298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T07:15:14,380 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/1361bde36c424f889f39ae2966fe36f3 2024-11-17T07:15:14,387 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-17T07:15:14,387 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-17T07:15:14,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/7142dedf087946e7b8f6fb84bd460d45 is 50, key is test_row_0/B:col10/1731827711925/Put/seqid=0 2024-11-17T07:15:14,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742231_1407 (size=12201) 2024-11-17T07:15:14,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827774507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,512 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:14,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827774517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:14,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:14,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827774518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,534 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:14,791 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/7142dedf087946e7b8f6fb84bd460d45 2024-11-17T07:15:14,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/13e390661aed4ea7a27c681047bbd8e0 is 50, key is test_row_0/C:col10/1731827711925/Put/seqid=0 2024-11-17T07:15:14,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742232_1408 (size=12201) 2024-11-17T07:15:14,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:14,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827774817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,829 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:14,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827774837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:14,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827774837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:14,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:15,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827775107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:15,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827775107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:15,119 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in d0c00ab4c3958615a6e7b967414841cf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,119 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in d0c00ab4c3958615a6e7b967414841cf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:15,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,120 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:15,203 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/13e390661aed4ea7a27c681047bbd8e0 2024-11-17T07:15:15,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/1361bde36c424f889f39ae2966fe36f3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/1361bde36c424f889f39ae2966fe36f3 2024-11-17T07:15:15,211 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/1361bde36c424f889f39ae2966fe36f3, entries=150, sequenceid=261, filesize=30.4 K 2024-11-17T07:15:15,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/7142dedf087946e7b8f6fb84bd460d45 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7142dedf087946e7b8f6fb84bd460d45 2024-11-17T07:15:15,215 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7142dedf087946e7b8f6fb84bd460d45, entries=150, sequenceid=261, filesize=11.9 K 2024-11-17T07:15:15,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/13e390661aed4ea7a27c681047bbd8e0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/13e390661aed4ea7a27c681047bbd8e0 2024-11-17T07:15:15,219 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/13e390661aed4ea7a27c681047bbd8e0, entries=150, sequenceid=261, filesize=11.9 K 2024-11-17T07:15:15,220 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for d0c00ab4c3958615a6e7b967414841cf in 1672ms, sequenceid=261, compaction requested=true 2024-11-17T07:15:15,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:15,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:15,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-17T07:15:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-17T07:15:15,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-17T07:15:15,223 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0500 sec 2024-11-17T07:15:15,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 3.0550 sec 2024-11-17T07:15:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:15,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T07:15:15,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:15,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:15,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:15,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:15,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:15,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:15,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111769a5623a61894dffab49d93578f753bb_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827714178/Put/seqid=0 2024-11-17T07:15:15,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742233_1409 (size=14994) 2024-11-17T07:15:15,360 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:15,365 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111769a5623a61894dffab49d93578f753bb_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111769a5623a61894dffab49d93578f753bb_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:15,366 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/72e34d99cea64d21bc01ff6b56ad6676, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:15,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/72e34d99cea64d21bc01ff6b56ad6676 is 175, key is test_row_0/A:col10/1731827714178/Put/seqid=0 2024-11-17T07:15:15,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742234_1410 (size=39949) 2024-11-17T07:15:15,377 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/72e34d99cea64d21bc01ff6b56ad6676 2024-11-17T07:15:15,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/0647991d4e6446bfaf454330e5ef5a9d is 50, key is test_row_0/B:col10/1731827714178/Put/seqid=0 2024-11-17T07:15:15,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742235_1411 (size=12301) 2024-11-17T07:15:15,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827775409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:15,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827775410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:15,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827775422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:15,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827775537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:15,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827775537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827775537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:15,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827775757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:15,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827775757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:15,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827775757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:15,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/0647991d4e6446bfaf454330e5ef5a9d 2024-11-17T07:15:15,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/9c04ebde36ff469e95a2031195041343 is 50, key is test_row_0/C:col10/1731827714178/Put/seqid=0 2024-11-17T07:15:15,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742236_1412 (size=12301) 2024-11-17T07:15:16,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827776077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:16,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827776077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:16,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827776078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:16,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/9c04ebde36ff469e95a2031195041343 2024-11-17T07:15:16,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/72e34d99cea64d21bc01ff6b56ad6676 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/72e34d99cea64d21bc01ff6b56ad6676 2024-11-17T07:15:16,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/72e34d99cea64d21bc01ff6b56ad6676, entries=200, sequenceid=273, filesize=39.0 K 2024-11-17T07:15:16,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/0647991d4e6446bfaf454330e5ef5a9d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/0647991d4e6446bfaf454330e5ef5a9d 2024-11-17T07:15:16,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/0647991d4e6446bfaf454330e5ef5a9d, entries=150, sequenceid=273, filesize=12.0 K 2024-11-17T07:15:16,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/9c04ebde36ff469e95a2031195041343 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/9c04ebde36ff469e95a2031195041343 2024-11-17T07:15:16,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/9c04ebde36ff469e95a2031195041343, entries=150, sequenceid=273, filesize=12.0 K 2024-11-17T07:15:16,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for d0c00ab4c3958615a6e7b967414841cf in 890ms, sequenceid=273, compaction requested=true 2024-11-17T07:15:16,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:16,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:16,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:16,232 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:15:16,232 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:15:16,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:16,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:16,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:16,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:16,233 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159482 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:15:16,233 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49044 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:15:16,233 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/A is initiating minor compaction (all files) 2024-11-17T07:15:16,233 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/B is initiating minor compaction (all files) 2024-11-17T07:15:16,233 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/A in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:16,233 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/B in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:16,234 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/43047641aadb4f73b0ef3459bd4bdbec, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/da0fa3b02fde4528b13cfa0f57f40dd5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/1361bde36c424f889f39ae2966fe36f3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/72e34d99cea64d21bc01ff6b56ad6676] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=155.7 K 2024-11-17T07:15:16,234 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/f8a55aa1935146fd8e5ccb917a9cb713, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/f19ddb9e87f14f17b710757ae2b38b54, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7142dedf087946e7b8f6fb84bd460d45, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/0647991d4e6446bfaf454330e5ef5a9d] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=47.9 K 2024-11-17T07:15:16,234 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:16,234 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/43047641aadb4f73b0ef3459bd4bdbec, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/da0fa3b02fde4528b13cfa0f57f40dd5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/1361bde36c424f889f39ae2966fe36f3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/72e34d99cea64d21bc01ff6b56ad6676] 2024-11-17T07:15:16,234 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43047641aadb4f73b0ef3459bd4bdbec, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731827710171 2024-11-17T07:15:16,234 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting f8a55aa1935146fd8e5ccb917a9cb713, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731827710171 2024-11-17T07:15:16,234 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting f19ddb9e87f14f17b710757ae2b38b54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731827710878 2024-11-17T07:15:16,234 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting da0fa3b02fde4528b13cfa0f57f40dd5, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731827710869 2024-11-17T07:15:16,235 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 7142dedf087946e7b8f6fb84bd460d45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731827711925 2024-11-17T07:15:16,235 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1361bde36c424f889f39ae2966fe36f3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731827711925 2024-11-17T07:15:16,235 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 0647991d4e6446bfaf454330e5ef5a9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731827714165 2024-11-17T07:15:16,235 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72e34d99cea64d21bc01ff6b56ad6676, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731827714165 2024-11-17T07:15:16,243 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#B#compaction#351 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:16,243 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/0ea84767dfd448218e4ae9ba61358b30 is 50, key is test_row_0/B:col10/1731827714178/Put/seqid=0 2024-11-17T07:15:16,244 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:16,249 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111782f40a86c6634431b4251fad67bfdf40_d0c00ab4c3958615a6e7b967414841cf store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:16,251 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111782f40a86c6634431b4251fad67bfdf40_d0c00ab4c3958615a6e7b967414841cf, store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:16,251 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111782f40a86c6634431b4251fad67bfdf40_d0c00ab4c3958615a6e7b967414841cf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:16,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742237_1413 (size=12677) 2024-11-17T07:15:16,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742238_1414 (size=4469) 2024-11-17T07:15:16,264 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#A#compaction#352 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:16,264 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/29ac3d3069c449cb9ac8a0a9cbf68bb1 is 175, key is test_row_0/A:col10/1731827714178/Put/seqid=0 2024-11-17T07:15:16,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742239_1415 (size=31631) 2024-11-17T07:15:16,275 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/29ac3d3069c449cb9ac8a0a9cbf68bb1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/29ac3d3069c449cb9ac8a0a9cbf68bb1 2024-11-17T07:15:16,279 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/A of d0c00ab4c3958615a6e7b967414841cf into 29ac3d3069c449cb9ac8a0a9cbf68bb1(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:16,279 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:16,279 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/A, priority=12, startTime=1731827716232; duration=0sec 2024-11-17T07:15:16,279 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:16,279 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:A 2024-11-17T07:15:16,279 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:15:16,280 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49044 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:15:16,280 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/C is initiating minor compaction (all files) 2024-11-17T07:15:16,280 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/C in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:16,281 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/062e459681234604aac8c859169f2808, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5ba39c79161a4915b87af676c2dcba11, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/13e390661aed4ea7a27c681047bbd8e0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/9c04ebde36ff469e95a2031195041343] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=47.9 K 2024-11-17T07:15:16,281 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 062e459681234604aac8c859169f2808, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731827710171 2024-11-17T07:15:16,282 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ba39c79161a4915b87af676c2dcba11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731827710878 2024-11-17T07:15:16,282 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13e390661aed4ea7a27c681047bbd8e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731827711925 2024-11-17T07:15:16,282 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c04ebde36ff469e95a2031195041343, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731827714165 2024-11-17T07:15:16,290 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:16,291 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/bea149b622154a539b3697354dcf0f76 is 50, key is test_row_0/C:col10/1731827714178/Put/seqid=0 2024-11-17T07:15:16,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742240_1416 (size=12677) 2024-11-17T07:15:16,307 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/bea149b622154a539b3697354dcf0f76 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/bea149b622154a539b3697354dcf0f76 2024-11-17T07:15:16,311 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/C of d0c00ab4c3958615a6e7b967414841cf into bea149b622154a539b3697354dcf0f76(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:16,311 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:16,311 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/C, priority=12, startTime=1731827716233; duration=0sec 2024-11-17T07:15:16,311 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:16,311 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:C 2024-11-17T07:15:16,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-17T07:15:16,337 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:16,337 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-17T07:15:16,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T07:15:16,339 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:16,339 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:16,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T07:15:16,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-17T07:15:16,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:16,492 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-17T07:15:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:16,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117edd006918c124a148bcea9cc90032eb5_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827715406/Put/seqid=0 2024-11-17T07:15:16,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742241_1417 (size=12454) 2024-11-17T07:15:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:16,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:16,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827776618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:16,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827776619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,632 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:16,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827776624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T07:15:16,658 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/0ea84767dfd448218e4ae9ba61358b30 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/0ea84767dfd448218e4ae9ba61358b30 2024-11-17T07:15:16,664 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/B of d0c00ab4c3958615a6e7b967414841cf into 0ea84767dfd448218e4ae9ba61358b30(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:16,664 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:16,664 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/B, priority=12, startTime=1731827716232; duration=0sec 2024-11-17T07:15:16,665 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:16,665 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:B 2024-11-17T07:15:16,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827776737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:16,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827776737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:16,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827776737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:16,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:16,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:16,910 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117edd006918c124a148bcea9cc90032eb5_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117edd006918c124a148bcea9cc90032eb5_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:16,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/a56558b2d1a44f0a9769a9b149978308, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:16,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/a56558b2d1a44f0a9769a9b149978308 is 175, key is test_row_0/A:col10/1731827715406/Put/seqid=0 2024-11-17T07:15:16,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742242_1418 (size=31255) 2024-11-17T07:15:16,935 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/a56558b2d1a44f0a9769a9b149978308 2024-11-17T07:15:16,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/7d3acb90ba86494aabcb3531f9b64384 is 50, key is test_row_0/B:col10/1731827715406/Put/seqid=0 2024-11-17T07:15:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827776948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:16,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:16,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827776948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:16,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:16,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827776949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:16,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:16,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742243_1419 (size=12301) 2024-11-17T07:15:16,966 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/7d3acb90ba86494aabcb3531f9b64384 2024-11-17T07:15:16,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T07:15:16,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/5bd751fbf5c94a2fbe407b9591bd2a4e is 50, key is test_row_0/C:col10/1731827715406/Put/seqid=0 2024-11-17T07:15:16,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742244_1420 (size=12301) 2024-11-17T07:15:16,988 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/5bd751fbf5c94a2fbe407b9591bd2a4e 2024-11-17T07:15:16,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/a56558b2d1a44f0a9769a9b149978308 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a56558b2d1a44f0a9769a9b149978308 2024-11-17T07:15:16,998 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a56558b2d1a44f0a9769a9b149978308, entries=150, sequenceid=295, filesize=30.5 K 2024-11-17T07:15:16,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/7d3acb90ba86494aabcb3531f9b64384 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7d3acb90ba86494aabcb3531f9b64384 2024-11-17T07:15:17,003 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7d3acb90ba86494aabcb3531f9b64384, entries=150, sequenceid=295, filesize=12.0 K 2024-11-17T07:15:17,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/5bd751fbf5c94a2fbe407b9591bd2a4e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5bd751fbf5c94a2fbe407b9591bd2a4e 2024-11-17T07:15:17,008 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5bd751fbf5c94a2fbe407b9591bd2a4e, entries=150, sequenceid=295, filesize=12.0 K 2024-11-17T07:15:17,009 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for d0c00ab4c3958615a6e7b967414841cf in 517ms, sequenceid=295, compaction requested=false 2024-11-17T07:15:17,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:17,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:17,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-17T07:15:17,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-17T07:15:17,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-17T07:15:17,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 672 msec 2024-11-17T07:15:17,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 684 msec 2024-11-17T07:15:17,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:17,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T07:15:17,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:17,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:17,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:17,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:17,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:17,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:17,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c82471c3b68747ea915fb981d97b7053_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827716616/Put/seqid=0 2024-11-17T07:15:17,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742245_1421 (size=14994) 2024-11-17T07:15:17,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827777335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:17,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:17,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827777335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:17,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827777341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:17,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:17,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827777457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:17,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:17,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827777458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:17,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827777458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-17T07:15:17,477 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:17,477 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-17T07:15:17,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T07:15:17,479 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:17,480 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:17,480 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T07:15:17,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T07:15:17,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:17,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:17,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:17,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:17,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:17,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:17,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827777667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:17,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827777667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:17,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827777667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:17,698 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:17,702 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117c82471c3b68747ea915fb981d97b7053_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c82471c3b68747ea915fb981d97b7053_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:17,702 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/0787f6d2047049ae8ee25d3876c2b8e6, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:17,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/0787f6d2047049ae8ee25d3876c2b8e6 is 175, key is test_row_0/A:col10/1731827716616/Put/seqid=0 2024-11-17T07:15:17,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742246_1422 (size=39949) 2024-11-17T07:15:17,707 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=309, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/0787f6d2047049ae8ee25d3876c2b8e6 2024-11-17T07:15:17,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/a0b59bb115b7451981b50516c247a197 is 50, key is test_row_0/B:col10/1731827716616/Put/seqid=0 2024-11-17T07:15:17,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742247_1423 (size=12301) 2024-11-17T07:15:17,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T07:15:17,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:17,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:17,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:17,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:17,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:17,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T07:15:17,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T07:15:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:17,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:17,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:17,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827777977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:17,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827777987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:17,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827777987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:17,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:18,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T07:15:18,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:18,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:18,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:18,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:18,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T07:15:18,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/a0b59bb115b7451981b50516c247a197 2024-11-17T07:15:18,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/85d9e2cc98fb4f4a9cbd9001899f1173 is 50, key is test_row_0/C:col10/1731827716616/Put/seqid=0 2024-11-17T07:15:18,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742248_1424 (size=12301) 2024-11-17T07:15:18,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T07:15:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:18,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:18,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:18,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T07:15:18,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:18,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:18,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:18,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:18,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:18,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:18,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827778497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:18,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:18,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827778497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:18,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:18,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:18,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:18,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827778507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:18,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:18,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:18,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:18,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/85d9e2cc98fb4f4a9cbd9001899f1173 2024-11-17T07:15:18,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/0787f6d2047049ae8ee25d3876c2b8e6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/0787f6d2047049ae8ee25d3876c2b8e6 2024-11-17T07:15:18,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/0787f6d2047049ae8ee25d3876c2b8e6, entries=200, sequenceid=309, filesize=39.0 K 2024-11-17T07:15:18,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/a0b59bb115b7451981b50516c247a197 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a0b59bb115b7451981b50516c247a197 2024-11-17T07:15:18,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a0b59bb115b7451981b50516c247a197, entries=150, sequenceid=309, filesize=12.0 K 2024-11-17T07:15:18,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/85d9e2cc98fb4f4a9cbd9001899f1173 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/85d9e2cc98fb4f4a9cbd9001899f1173 2024-11-17T07:15:18,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/85d9e2cc98fb4f4a9cbd9001899f1173, entries=150, sequenceid=309, filesize=12.0 K 2024-11-17T07:15:18,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for d0c00ab4c3958615a6e7b967414841cf in 1276ms, sequenceid=309, compaction requested=true 2024-11-17T07:15:18,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:18,547 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:18,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:18,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:18,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:18,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:18,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:18,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:18,547 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:18,548 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:18,548 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/A is initiating minor compaction (all files) 2024-11-17T07:15:18,548 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/A in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:18,548 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/29ac3d3069c449cb9ac8a0a9cbf68bb1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a56558b2d1a44f0a9769a9b149978308, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/0787f6d2047049ae8ee25d3876c2b8e6] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=100.4 K 2024-11-17T07:15:18,548 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:18,548 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:18,548 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/29ac3d3069c449cb9ac8a0a9cbf68bb1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a56558b2d1a44f0a9769a9b149978308, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/0787f6d2047049ae8ee25d3876c2b8e6] 2024-11-17T07:15:18,549 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/B is initiating minor compaction (all files) 2024-11-17T07:15:18,549 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/B in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:18,549 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/0ea84767dfd448218e4ae9ba61358b30, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7d3acb90ba86494aabcb3531f9b64384, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a0b59bb115b7451981b50516c247a197] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=36.4 K 2024-11-17T07:15:18,549 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29ac3d3069c449cb9ac8a0a9cbf68bb1, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731827714165 2024-11-17T07:15:18,549 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ea84767dfd448218e4ae9ba61358b30, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731827714165 2024-11-17T07:15:18,549 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56558b2d1a44f0a9769a9b149978308, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731827715406 2024-11-17T07:15:18,550 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d3acb90ba86494aabcb3531f9b64384, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731827715406 2024-11-17T07:15:18,550 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0787f6d2047049ae8ee25d3876c2b8e6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731827716613 2024-11-17T07:15:18,550 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a0b59bb115b7451981b50516c247a197, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731827716616 2024-11-17T07:15:18,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-17T07:15:18,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:18,551 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T07:15:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:18,556 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:18,557 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#B#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:18,558 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/be15eedf3cb34b4bad45c242e6d51205 is 50, key is test_row_0/B:col10/1731827716616/Put/seqid=0 2024-11-17T07:15:18,561 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117ef67f3f3792746ddb41b4ea1ad2fbe7a_d0c00ab4c3958615a6e7b967414841cf store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:18,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117eec3bd2ce9b847f3bc761be650b20daf_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827717340/Put/seqid=0 2024-11-17T07:15:18,563 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117ef67f3f3792746ddb41b4ea1ad2fbe7a_d0c00ab4c3958615a6e7b967414841cf, store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:18,563 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ef67f3f3792746ddb41b4ea1ad2fbe7a_d0c00ab4c3958615a6e7b967414841cf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:18,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742250_1426 (size=12454) 2024-11-17T07:15:18,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742249_1425 (size=12779) 2024-11-17T07:15:18,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742251_1427 (size=4469) 2024-11-17T07:15:18,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T07:15:18,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:18,981 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117eec3bd2ce9b847f3bc761be650b20daf_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eec3bd2ce9b847f3bc761be650b20daf_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:18,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/5429a402a3aa4d11a36d116458dc7108, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:18,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/5429a402a3aa4d11a36d116458dc7108 is 175, key is test_row_0/A:col10/1731827717340/Put/seqid=0 2024-11-17T07:15:18,983 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/be15eedf3cb34b4bad45c242e6d51205 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/be15eedf3cb34b4bad45c242e6d51205 2024-11-17T07:15:18,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742252_1428 (size=31255) 2024-11-17T07:15:18,986 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/5429a402a3aa4d11a36d116458dc7108 2024-11-17T07:15:18,989 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/B of d0c00ab4c3958615a6e7b967414841cf into be15eedf3cb34b4bad45c242e6d51205(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:18,989 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:18,989 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/B, priority=13, startTime=1731827718547; duration=0sec 2024-11-17T07:15:18,989 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:18,990 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:B 2024-11-17T07:15:18,990 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:18,992 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#A#compaction#360 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:18,992 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/9b6055982542489ba0d36faaf247c0d4 is 175, key is test_row_0/A:col10/1731827716616/Put/seqid=0 2024-11-17T07:15:18,993 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37279 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:18,993 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/C is initiating minor compaction (all files) 2024-11-17T07:15:18,993 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/C in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:18,993 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/bea149b622154a539b3697354dcf0f76, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5bd751fbf5c94a2fbe407b9591bd2a4e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/85d9e2cc98fb4f4a9cbd9001899f1173] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=36.4 K 2024-11-17T07:15:18,994 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting bea149b622154a539b3697354dcf0f76, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731827714165 2024-11-17T07:15:18,994 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd751fbf5c94a2fbe407b9591bd2a4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731827715406 2024-11-17T07:15:18,994 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 85d9e2cc98fb4f4a9cbd9001899f1173, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731827716616 2024-11-17T07:15:18,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/417a3307d0014c4c86eef3141d217548 is 50, key is test_row_0/B:col10/1731827717340/Put/seqid=0 2024-11-17T07:15:19,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742253_1429 (size=31733) 2024-11-17T07:15:19,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742254_1430 (size=12301) 2024-11-17T07:15:19,014 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#C#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:19,015 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/8ebde51a56e940b0a3447304fc1f3055 is 50, key is test_row_0/C:col10/1731827716616/Put/seqid=0 2024-11-17T07:15:19,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742255_1431 (size=12779) 2024-11-17T07:15:19,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:19,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:19,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:19,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827779177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:19,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:19,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:19,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827779178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:19,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,194 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:19,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:19,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827779287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,296 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:19,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827779296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:19,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:19,405 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/9b6055982542489ba0d36faaf247c0d4 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/9b6055982542489ba0d36faaf247c0d4 2024-11-17T07:15:19,410 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/A of d0c00ab4c3958615a6e7b967414841cf into 9b6055982542489ba0d36faaf247c0d4(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:19,410 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:19,410 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/A, priority=13, startTime=1731827718547; duration=0sec 2024-11-17T07:15:19,410 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:19,410 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:A 2024-11-17T07:15:19,410 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/417a3307d0014c4c86eef3141d217548 2024-11-17T07:15:19,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/e7effd6d2b4e4380af607f7f33d3e5fe is 50, key is test_row_0/C:col10/1731827717340/Put/seqid=0 2024-11-17T07:15:19,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742256_1432 (size=12301) 2024-11-17T07:15:19,426 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/8ebde51a56e940b0a3447304fc1f3055 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/8ebde51a56e940b0a3447304fc1f3055 2024-11-17T07:15:19,431 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/C of d0c00ab4c3958615a6e7b967414841cf into 8ebde51a56e940b0a3447304fc1f3055(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:19,431 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:19,431 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/C, priority=13, startTime=1731827718547; duration=0sec 2024-11-17T07:15:19,431 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:19,431 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:C 2024-11-17T07:15:19,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:19,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827779507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:19,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:19,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827779507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:19,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827779507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:19,514 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in d0c00ab4c3958615a6e7b967414841cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:19,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:19,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827779517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:19,527 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in d0c00ab4c3958615a6e7b967414841cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:19,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:19,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:19,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827779527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:19,533 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in d0c00ab4c3958615a6e7b967414841cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T07:15:19,821 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/e7effd6d2b4e4380af607f7f33d3e5fe 2024-11-17T07:15:19,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/5429a402a3aa4d11a36d116458dc7108 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5429a402a3aa4d11a36d116458dc7108 2024-11-17T07:15:19,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827779817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827779817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:19,829 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5429a402a3aa4d11a36d116458dc7108, entries=150, sequenceid=328, filesize=30.5 K 2024-11-17T07:15:19,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/417a3307d0014c4c86eef3141d217548 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/417a3307d0014c4c86eef3141d217548 2024-11-17T07:15:19,834 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/417a3307d0014c4c86eef3141d217548, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T07:15:19,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/e7effd6d2b4e4380af607f7f33d3e5fe as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/e7effd6d2b4e4380af607f7f33d3e5fe 2024-11-17T07:15:19,839 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/e7effd6d2b4e4380af607f7f33d3e5fe, entries=150, sequenceid=328, filesize=12.0 K 2024-11-17T07:15:19,840 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for d0c00ab4c3958615a6e7b967414841cf in 1290ms, sequenceid=328, compaction requested=false 2024-11-17T07:15:19,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:19,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:19,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-17T07:15:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-17T07:15:19,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-17T07:15:19,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3620 sec 2024-11-17T07:15:19,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.3670 sec 2024-11-17T07:15:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:20,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T07:15:20,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:20,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:20,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:20,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:20,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:20,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:20,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117818dbc0478604052bd5a43e8891a42f4_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827720343/Put/seqid=0 2024-11-17T07:15:20,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742257_1433 (size=17534) 2024-11-17T07:15:20,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827780395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:20,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:20,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:20,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827780399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:20,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:20,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:20,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:20,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827780507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:20,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:20,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827780517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:20,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:20,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:20,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:20,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827780717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:20,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827780738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:20,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:20,756 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:20,760 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117818dbc0478604052bd5a43e8891a42f4_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117818dbc0478604052bd5a43e8891a42f4_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:20,761 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/c750205c6db34319bd8a7f8fc835b92c, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:20,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/c750205c6db34319bd8a7f8fc835b92c is 175, key is test_row_0/A:col10/1731827720343/Put/seqid=0 2024-11-17T07:15:20,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742258_1434 (size=48639) 2024-11-17T07:15:21,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:21,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827781027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:21,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827781047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:21,165 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/c750205c6db34319bd8a7f8fc835b92c 2024-11-17T07:15:21,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/9b593991d6404552b82893f52b97d538 is 50, key is test_row_0/B:col10/1731827720343/Put/seqid=0 2024-11-17T07:15:21,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742259_1435 (size=12301) 2024-11-17T07:15:21,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36564 deadline: 1731827781537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:21,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827781537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:21,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:21,542 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in d0c00ab4c3958615a6e7b967414841cf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,543 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:21,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36502 deadline: 1731827781547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:21,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36550 deadline: 1731827781547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:21,554 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in d0c00ab4c3958615a6e7b967414841cf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:21,554 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in d0c00ab4c3958615a6e7b967414841cf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:21,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827781557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:21,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/9b593991d6404552b82893f52b97d538 2024-11-17T07:15:21,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/f371f4e24eef45b5bde0c41a9646e7bc is 50, key is test_row_0/C:col10/1731827720343/Put/seqid=0 2024-11-17T07:15:21,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742260_1436 (size=12301) 2024-11-17T07:15:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-17T07:15:21,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:21,648 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-17T07:15:21,650 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T07:15:21,650 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:21,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T07:15:21,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T07:15:21,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:21,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:21,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:21,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:21,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:21,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T07:15:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:21,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:21,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T07:15:21,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/f371f4e24eef45b5bde0c41a9646e7bc 2024-11-17T07:15:21,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/c750205c6db34319bd8a7f8fc835b92c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/c750205c6db34319bd8a7f8fc835b92c 2024-11-17T07:15:21,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/c750205c6db34319bd8a7f8fc835b92c, entries=250, sequenceid=346, filesize=47.5 K 2024-11-17T07:15:22,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/9b593991d6404552b82893f52b97d538 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/9b593991d6404552b82893f52b97d538 2024-11-17T07:15:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/9b593991d6404552b82893f52b97d538, entries=150, sequenceid=346, filesize=12.0 K 2024-11-17T07:15:22,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/f371f4e24eef45b5bde0c41a9646e7bc as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/f371f4e24eef45b5bde0c41a9646e7bc 2024-11-17T07:15:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/f371f4e24eef45b5bde0c41a9646e7bc, entries=150, sequenceid=346, filesize=12.0 K 2024-11-17T07:15:22,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for d0c00ab4c3958615a6e7b967414841cf in 1667ms, sequenceid=346, compaction requested=true 2024-11-17T07:15:22,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:22,013 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:22,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:22,013 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:22,013 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:22,013 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:22,013 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/A is initiating minor compaction (all files) 2024-11-17T07:15:22,014 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/B is initiating minor compaction (all files) 2024-11-17T07:15:22,014 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/A in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:22,014 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/B in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:22,014 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/9b6055982542489ba0d36faaf247c0d4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5429a402a3aa4d11a36d116458dc7108, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/c750205c6db34319bd8a7f8fc835b92c] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=109.0 K 2024-11-17T07:15:22,014 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/be15eedf3cb34b4bad45c242e6d51205, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/417a3307d0014c4c86eef3141d217548, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/9b593991d6404552b82893f52b97d538] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=36.5 K 2024-11-17T07:15:22,014 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:22,014 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/9b6055982542489ba0d36faaf247c0d4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5429a402a3aa4d11a36d116458dc7108, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/c750205c6db34319bd8a7f8fc835b92c] 2024-11-17T07:15:22,014 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b6055982542489ba0d36faaf247c0d4, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731827716616 2024-11-17T07:15:22,014 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting be15eedf3cb34b4bad45c242e6d51205, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731827716616 2024-11-17T07:15:22,014 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5429a402a3aa4d11a36d116458dc7108, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731827717333 2024-11-17T07:15:22,014 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 417a3307d0014c4c86eef3141d217548, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731827717333 2024-11-17T07:15:22,015 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting c750205c6db34319bd8a7f8fc835b92c, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731827719161 2024-11-17T07:15:22,015 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b593991d6404552b82893f52b97d538, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731827719161 2024-11-17T07:15:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,025 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:22,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,029 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#B#compaction#369 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,029 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/b89410487e72436e88dc6cf77327698f is 50, key is test_row_0/B:col10/1731827720343/Put/seqid=0 2024-11-17T07:15:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,030 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111735d5f19d471245bf9ced60ed0ca54e75_d0c00ab4c3958615a6e7b967414841cf store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,035 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111735d5f19d471245bf9ced60ed0ca54e75_d0c00ab4c3958615a6e7b967414841cf, store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:22,036 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111735d5f19d471245bf9ced60ed0ca54e75_d0c00ab4c3958615a6e7b967414841cf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742261_1437 (size=12881) 2024-11-17T07:15:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,066 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/b89410487e72436e88dc6cf77327698f as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b89410487e72436e88dc6cf77327698f 2024-11-17T07:15:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742262_1438 (size=4469) 2024-11-17T07:15:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,071 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#A#compaction#370 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,072 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/8647fb43c0c14c5bac0ae19ea7436a7d is 175, key is test_row_0/A:col10/1731827720343/Put/seqid=0 2024-11-17T07:15:22,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,076 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/B of d0c00ab4c3958615a6e7b967414841cf into b89410487e72436e88dc6cf77327698f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:22,076 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:22,076 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/B, priority=13, startTime=1731827722013; duration=0sec 2024-11-17T07:15:22,076 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:22,076 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:B 2024-11-17T07:15:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,077 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,078 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:22,078 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/C is initiating minor compaction (all files) 2024-11-17T07:15:22,078 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/C in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:22,078 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/8ebde51a56e940b0a3447304fc1f3055, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/e7effd6d2b4e4380af607f7f33d3e5fe, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/f371f4e24eef45b5bde0c41a9646e7bc] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=36.5 K 2024-11-17T07:15:22,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,078 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ebde51a56e940b0a3447304fc1f3055, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1731827716616 2024-11-17T07:15:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,079 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting e7effd6d2b4e4380af607f7f33d3e5fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731827717333 2024-11-17T07:15:22,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,079 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting f371f4e24eef45b5bde0c41a9646e7bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731827719161 2024-11-17T07:15:22,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742263_1439 (size=31835) 2024-11-17T07:15:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,084 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/8647fb43c0c14c5bac0ae19ea7436a7d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/8647fb43c0c14c5bac0ae19ea7436a7d 2024-11-17T07:15:22,091 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#C#compaction#371 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:22,092 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/5c98d41a2815474a8727ccf617145e70 is 50, key is test_row_0/C:col10/1731827720343/Put/seqid=0 2024-11-17T07:15:22,095 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/A of d0c00ab4c3958615a6e7b967414841cf into 8647fb43c0c14c5bac0ae19ea7436a7d(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:22,095 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:22,095 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/A, priority=13, startTime=1731827722013; duration=0sec 2024-11-17T07:15:22,095 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:22,095 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:A 2024-11-17T07:15:22,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742264_1440 (size=12881) 2024-11-17T07:15:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-17T07:15:22,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,110 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T07:15:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:22,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:22,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:22,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:22,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:22,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,115 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/5c98d41a2815474a8727ccf617145e70 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5c98d41a2815474a8727ccf617145e70 2024-11-17T07:15:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,120 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/C of d0c00ab4c3958615a6e7b967414841cf into 5c98d41a2815474a8727ccf617145e70(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,120 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:22,120 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/C, priority=13, startTime=1731827722013; duration=0sec 2024-11-17T07:15:22,120 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:22,120 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:C 2024-11-17T07:15:22,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a169cc591246442ab6824f200c65ba7a_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827720397/Put/seqid=0 2024-11-17T07:15:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742265_1441 (size=9914) 2024-11-17T07:15:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,145 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117a169cc591246442ab6824f200c65ba7a_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a169cc591246442ab6824f200c65ba7a_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:22,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/d2799e192c284d36b39e19e1215d0f18, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:22,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/d2799e192c284d36b39e19e1215d0f18 is 175, key is test_row_0/A:col10/1731827720397/Put/seqid=0 2024-11-17T07:15:22,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742266_1442 (size=22561) 2024-11-17T07:15:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T07:15:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,554 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/d2799e192c284d36b39e19e1215d0f18 2024-11-17T07:15:22,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/469baded619845fc96a496d2554c2eb3 is 50, key is test_row_0/B:col10/1731827720397/Put/seqid=0 2024-11-17T07:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742267_1443 (size=9857) 2024-11-17T07:15:22,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,576 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/469baded619845fc96a496d2554c2eb3 2024-11-17T07:15:22,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/3bc55412ce994951ab1c610c78ae1356 is 50, key is test_row_0/C:col10/1731827720397/Put/seqid=0 2024-11-17T07:15:22,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742268_1444 (size=9857) 2024-11-17T07:15:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:22,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:22,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:22,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827782693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:22,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:22,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827782695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:22,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T07:15:22,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:22,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827782807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:22,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:22,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:22,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:22,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827782807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:22,814 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:22,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:22,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:22,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:23,009 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/3bc55412ce994951ab1c610c78ae1356 2024-11-17T07:15:23,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/d2799e192c284d36b39e19e1215d0f18 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/d2799e192c284d36b39e19e1215d0f18 2024-11-17T07:15:23,019 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/d2799e192c284d36b39e19e1215d0f18, entries=100, sequenceid=365, filesize=22.0 K 2024-11-17T07:15:23,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/469baded619845fc96a496d2554c2eb3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/469baded619845fc96a496d2554c2eb3 2024-11-17T07:15:23,024 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/469baded619845fc96a496d2554c2eb3, entries=100, sequenceid=365, filesize=9.6 K 2024-11-17T07:15:23,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827783017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:23,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827783017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:23,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/3bc55412ce994951ab1c610c78ae1356 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3bc55412ce994951ab1c610c78ae1356 2024-11-17T07:15:23,031 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3bc55412ce994951ab1c610c78ae1356, entries=100, sequenceid=365, filesize=9.6 K 2024-11-17T07:15:23,032 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for d0c00ab4c3958615a6e7b967414841cf in 922ms, sequenceid=365, compaction requested=false 2024-11-17T07:15:23,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:23,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:23,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-17T07:15:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-17T07:15:23,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-17T07:15:23,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3830 sec 2024-11-17T07:15:23,037 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.3880 sec 2024-11-17T07:15:23,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:23,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T07:15:23,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:23,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:23,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:23,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:23,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:23,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:23,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111757ede32796984681ba5543b43bc3bdc3_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827722693/Put/seqid=0 2024-11-17T07:15:23,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742269_1445 (size=14994) 2024-11-17T07:15:23,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827783386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:23,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:23,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827783393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:23,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:23,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827783497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:23,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:23,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827783507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:23,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827783707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:23,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827783727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:23,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:23,744 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:23,747 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111757ede32796984681ba5543b43bc3bdc3_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111757ede32796984681ba5543b43bc3bdc3_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:23,748 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/a6687d70f4cd4dac91287be31e33eed5, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:23,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/a6687d70f4cd4dac91287be31e33eed5 is 175, key is test_row_0/A:col10/1731827722693/Put/seqid=0 2024-11-17T07:15:23,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742270_1446 (size=39949) 2024-11-17T07:15:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-17T07:15:23,797 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:23,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-17T07:15:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T07:15:23,799 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:23,800 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:23,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:23,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T07:15:23,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T07:15:23,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:23,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:23,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:23,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:23,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:23,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827784027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:24,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:24,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827784047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:24,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:24,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T07:15:24,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:24,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T07:15:24,153 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=382, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/a6687d70f4cd4dac91287be31e33eed5 2024-11-17T07:15:24,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/b71e7c449c6e4cf4b0cf95fb475b03f5 is 50, key is test_row_0/B:col10/1731827722693/Put/seqid=0 2024-11-17T07:15:24,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742271_1447 (size=12301) 2024-11-17T07:15:24,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T07:15:24,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:24,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T07:15:24,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:24,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T07:15:24,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36576 deadline: 1731827784537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:24,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:24,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T07:15:24,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:24,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,565 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/b71e7c449c6e4cf4b0cf95fb475b03f5 2024-11-17T07:15:24,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36514 deadline: 1731827784567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:24,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:15:24,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/d5a7676dab1741cda5048e889ada0e27 is 50, key is test_row_0/C:col10/1731827722693/Put/seqid=0 2024-11-17T07:15:24,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742272_1448 (size=12301) 2024-11-17T07:15:24,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T07:15:24,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:24,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,724 INFO [Thread-1673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1673. 2024-11-17T07:15:24,724 DEBUG [Thread-1673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:24,724 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:24,724 DEBUG [Thread-1673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:24,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:24,724 INFO [Thread-1671 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1671. 2024-11-17T07:15:24,724 DEBUG [Thread-1671 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:24,724 DEBUG [Thread-1671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:24,724 DEBUG [Thread-1671 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:24,724 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:24,727 INFO [Thread-1669 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1669. 2024-11-17T07:15:24,727 DEBUG [Thread-1669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:24,727 DEBUG [Thread-1669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:24,727 DEBUG [Thread-1669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:24,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:24,734 INFO [Thread-1675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1675. 2024-11-17T07:15:24,734 DEBUG [Thread-1675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:24,734 DEBUG [Thread-1675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:24,734 DEBUG [Thread-1675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:24,734 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:24,735 INFO [Thread-1677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1677. 2024-11-17T07:15:24,735 DEBUG [Thread-1677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:24,735 DEBUG [Thread-1677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:24,735 DEBUG [Thread-1677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:24,735 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:24,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T07:15:24,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T07:15:24,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/d5a7676dab1741cda5048e889ada0e27 2024-11-17T07:15:24,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/a6687d70f4cd4dac91287be31e33eed5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a6687d70f4cd4dac91287be31e33eed5 2024-11-17T07:15:24,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a6687d70f4cd4dac91287be31e33eed5, entries=200, sequenceid=382, filesize=39.0 K 2024-11-17T07:15:24,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/b71e7c449c6e4cf4b0cf95fb475b03f5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b71e7c449c6e4cf4b0cf95fb475b03f5 2024-11-17T07:15:24,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b71e7c449c6e4cf4b0cf95fb475b03f5, entries=150, sequenceid=382, filesize=12.0 K 2024-11-17T07:15:24,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/d5a7676dab1741cda5048e889ada0e27 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/d5a7676dab1741cda5048e889ada0e27 2024-11-17T07:15:24,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/d5a7676dab1741cda5048e889ada0e27, entries=150, sequenceid=382, filesize=12.0 K 2024-11-17T07:15:24,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for d0c00ab4c3958615a6e7b967414841cf in 1665ms, sequenceid=382, compaction requested=true 2024-11-17T07:15:24,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:24,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:24,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:24,996 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:24,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:24,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:24,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d0c00ab4c3958615a6e7b967414841cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:24,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:24,996 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:24,997 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:24,997 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94345 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:24,997 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/A is initiating minor compaction (all files) 2024-11-17T07:15:24,997 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/B is initiating minor compaction (all files) 2024-11-17T07:15:24,997 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/A in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,997 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/B in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,997 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b89410487e72436e88dc6cf77327698f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/469baded619845fc96a496d2554c2eb3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b71e7c449c6e4cf4b0cf95fb475b03f5] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=34.2 K 2024-11-17T07:15:24,997 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/8647fb43c0c14c5bac0ae19ea7436a7d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/d2799e192c284d36b39e19e1215d0f18, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a6687d70f4cd4dac91287be31e33eed5] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=92.1 K 2024-11-17T07:15:24,997 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:24,997 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/8647fb43c0c14c5bac0ae19ea7436a7d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/d2799e192c284d36b39e19e1215d0f18, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a6687d70f4cd4dac91287be31e33eed5] 2024-11-17T07:15:24,997 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting b89410487e72436e88dc6cf77327698f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731827719161 2024-11-17T07:15:24,997 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8647fb43c0c14c5bac0ae19ea7436a7d, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731827719161 2024-11-17T07:15:24,998 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 469baded619845fc96a496d2554c2eb3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731827720381 2024-11-17T07:15:24,998 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2799e192c284d36b39e19e1215d0f18, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731827720381 2024-11-17T07:15:24,998 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6687d70f4cd4dac91287be31e33eed5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731827722685 2024-11-17T07:15:24,998 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting b71e7c449c6e4cf4b0cf95fb475b03f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731827722685 2024-11-17T07:15:25,003 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#B#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:25,004 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/cf70546c8c524a25a2bd9279a952a459 is 50, key is test_row_0/B:col10/1731827722693/Put/seqid=0 2024-11-17T07:15:25,006 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:25,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742273_1449 (size=12983) 2024-11-17T07:15:25,007 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117b812002d2e5c478dbcffca4ed383ae34_d0c00ab4c3958615a6e7b967414841cf store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:25,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-17T07:15:25,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:25,024 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-17T07:15:25,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:25,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:25,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:25,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:25,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:25,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:25,027 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117b812002d2e5c478dbcffca4ed383ae34_d0c00ab4c3958615a6e7b967414841cf, store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:25,027 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117b812002d2e5c478dbcffca4ed383ae34_d0c00ab4c3958615a6e7b967414841cf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:25,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742274_1450 (size=4469) 2024-11-17T07:15:25,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176dea00ba287847ea903c65dc6d006cd5_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827723375/Put/seqid=0 2024-11-17T07:15:25,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742275_1451 (size=12454) 2024-11-17T07:15:25,411 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/cf70546c8c524a25a2bd9279a952a459 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/cf70546c8c524a25a2bd9279a952a459 2024-11-17T07:15:25,415 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/B of d0c00ab4c3958615a6e7b967414841cf into cf70546c8c524a25a2bd9279a952a459(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:25,415 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:25,415 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/B, priority=13, startTime=1731827724996; duration=0sec 2024-11-17T07:15:25,415 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:25,415 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:B 2024-11-17T07:15:25,416 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:25,416 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:25,416 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): d0c00ab4c3958615a6e7b967414841cf/C is initiating minor compaction (all files) 2024-11-17T07:15:25,416 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d0c00ab4c3958615a6e7b967414841cf/C in TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:25,417 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5c98d41a2815474a8727ccf617145e70, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3bc55412ce994951ab1c610c78ae1356, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/d5a7676dab1741cda5048e889ada0e27] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp, totalSize=34.2 K 2024-11-17T07:15:25,417 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c98d41a2815474a8727ccf617145e70, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731827719161 2024-11-17T07:15:25,417 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bc55412ce994951ab1c610c78ae1356, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731827720381 2024-11-17T07:15:25,417 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting d5a7676dab1741cda5048e889ada0e27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=382, earliestPutTs=1731827722685 2024-11-17T07:15:25,423 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#C#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:25,423 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/ec3a4ebd872d4ece95245154012391e8 is 50, key is test_row_0/C:col10/1731827722693/Put/seqid=0 2024-11-17T07:15:25,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742276_1452 (size=12983) 2024-11-17T07:15:25,436 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d0c00ab4c3958615a6e7b967414841cf#A#compaction#379 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:25,437 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/47515bc88bea448ea66243571b1b7682 is 175, key is test_row_0/A:col10/1731827722693/Put/seqid=0 2024-11-17T07:15:25,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742277_1453 (size=31937) 2024-11-17T07:15:25,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:25,444 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176dea00ba287847ea903c65dc6d006cd5_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176dea00ba287847ea903c65dc6d006cd5_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:25,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/d010a282654c4ce9a43be2e88cde98bc, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:25,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/d010a282654c4ce9a43be2e88cde98bc is 175, key is test_row_0/A:col10/1731827723375/Put/seqid=0 2024-11-17T07:15:25,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742278_1454 (size=31255) 2024-11-17T07:15:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:25,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. as already flushing 2024-11-17T07:15:25,558 INFO [Thread-1658 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1658. 2024-11-17T07:15:25,558 DEBUG [Thread-1658 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:25,558 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:25,558 INFO [Thread-1666 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1666. 2024-11-17T07:15:25,558 DEBUG [Thread-1666 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:25,558 DEBUG [Thread-1666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:25,558 DEBUG [Thread-1658 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:25,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:25,559 DEBUG [Thread-1666 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:25,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:25,568 INFO [Thread-1660 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1660. 2024-11-17T07:15:25,568 DEBUG [Thread-1660 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:25,568 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:25,568 DEBUG [Thread-1660 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:25,568 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:25,578 INFO [Thread-1664 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1664. 2024-11-17T07:15:25,578 DEBUG [Thread-1664 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:25,578 DEBUG [Thread-1664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:25,578 DEBUG [Thread-1664 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:25,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:25,588 INFO [Thread-1662 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1662. 2024-11-17T07:15:25,588 DEBUG [Thread-1662 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:25,588 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:25,588 DEBUG [Thread-1662 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:25,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:25,834 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/ec3a4ebd872d4ece95245154012391e8 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/ec3a4ebd872d4ece95245154012391e8 2024-11-17T07:15:25,838 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/C of d0c00ab4c3958615a6e7b967414841cf into ec3a4ebd872d4ece95245154012391e8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:25,838 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:25,838 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/C, priority=13, startTime=1731827724996; duration=0sec 2024-11-17T07:15:25,838 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:25,838 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:C 2024-11-17T07:15:25,843 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/47515bc88bea448ea66243571b1b7682 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/47515bc88bea448ea66243571b1b7682 2024-11-17T07:15:25,847 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d0c00ab4c3958615a6e7b967414841cf/A of d0c00ab4c3958615a6e7b967414841cf into 47515bc88bea448ea66243571b1b7682(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:25,847 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:25,847 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., storeName=d0c00ab4c3958615a6e7b967414841cf/A, priority=13, startTime=1731827724996; duration=0sec 2024-11-17T07:15:25,847 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:25,847 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d0c00ab4c3958615a6e7b967414841cf:A 2024-11-17T07:15:25,849 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=398, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/d010a282654c4ce9a43be2e88cde98bc 2024-11-17T07:15:25,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/b4ef1c9fa84d4169b0e7f934014f1347 is 50, key is test_row_0/B:col10/1731827723375/Put/seqid=0 2024-11-17T07:15:25,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742279_1455 (size=12301) 2024-11-17T07:15:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T07:15:26,257 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/b4ef1c9fa84d4169b0e7f934014f1347 2024-11-17T07:15:26,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/44425bba8cde4900a4ce8a9e0aa79712 is 50, key is test_row_0/C:col10/1731827723375/Put/seqid=0 2024-11-17T07:15:26,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742280_1456 (size=12301) 2024-11-17T07:15:26,667 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/44425bba8cde4900a4ce8a9e0aa79712 2024-11-17T07:15:26,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/d010a282654c4ce9a43be2e88cde98bc as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/d010a282654c4ce9a43be2e88cde98bc 2024-11-17T07:15:26,673 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/d010a282654c4ce9a43be2e88cde98bc, entries=150, sequenceid=398, filesize=30.5 K 2024-11-17T07:15:26,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/b4ef1c9fa84d4169b0e7f934014f1347 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b4ef1c9fa84d4169b0e7f934014f1347 2024-11-17T07:15:26,676 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b4ef1c9fa84d4169b0e7f934014f1347, entries=150, sequenceid=398, filesize=12.0 K 2024-11-17T07:15:26,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/44425bba8cde4900a4ce8a9e0aa79712 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/44425bba8cde4900a4ce8a9e0aa79712 2024-11-17T07:15:26,679 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/44425bba8cde4900a4ce8a9e0aa79712, entries=150, sequenceid=398, filesize=12.0 K 2024-11-17T07:15:26,680 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=33.54 KB/34350 for d0c00ab4c3958615a6e7b967414841cf in 1656ms, sequenceid=398, compaction requested=false 2024-11-17T07:15:26,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for d0c00ab4c3958615a6e7b967414841cf: 2024-11-17T07:15:26,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:26,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-17T07:15:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-17T07:15:26,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-17T07:15:26,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8810 sec 2024-11-17T07:15:26,684 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.8850 sec 2024-11-17T07:15:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-17T07:15:27,967 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 997 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2984 rows 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1000 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2998 rows 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 996 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2984 rows 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 987 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2961 rows 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1003 2024-11-17T07:15:27,967 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2998 rows 2024-11-17T07:15:27,967 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T07:15:27,968 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T07:15:27,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:27,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:27,968 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:27,968 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T07:15:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T07:15:27,971 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827727971"}]},"ts":"1731827727971"} 2024-11-17T07:15:27,972 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T07:15:27,972 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T07:15:27,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T07:15:27,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, UNASSIGN}] 2024-11-17T07:15:27,975 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, UNASSIGN 2024-11-17T07:15:27,975 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=d0c00ab4c3958615a6e7b967414841cf, regionState=CLOSING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:27,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, UNASSIGN because future has completed 2024-11-17T07:15:27,977 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T07:15:27,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:15:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T07:15:28,129 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:28,129 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T07:15:28,129 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing d0c00ab4c3958615a6e7b967414841cf, disabling compactions & flushes 2024-11-17T07:15:28,129 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:28,129 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:28,129 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. after waiting 0 ms 2024-11-17T07:15:28,129 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:28,129 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing d0c00ab4c3958615a6e7b967414841cf 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T07:15:28,129 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=A 2024-11-17T07:15:28,129 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:28,129 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=B 2024-11-17T07:15:28,129 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:28,129 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK d0c00ab4c3958615a6e7b967414841cf, store=C 2024-11-17T07:15:28,129 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:28,134 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e5b95eb36c9e4b91bce6878ffb499dcd_d0c00ab4c3958615a6e7b967414841cf is 50, key is test_row_0/A:col10/1731827725567/Put/seqid=0 2024-11-17T07:15:28,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742281_1457 (size=12454) 2024-11-17T07:15:28,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T07:15:28,538 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:28,541 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e5b95eb36c9e4b91bce6878ffb499dcd_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e5b95eb36c9e4b91bce6878ffb499dcd_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:28,542 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/f9c69276fafd4f51801adc05eb470712, store: [table=TestAcidGuarantees family=A region=d0c00ab4c3958615a6e7b967414841cf] 2024-11-17T07:15:28,542 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/f9c69276fafd4f51801adc05eb470712 is 175, key is test_row_0/A:col10/1731827725567/Put/seqid=0 2024-11-17T07:15:28,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742282_1458 (size=31255) 2024-11-17T07:15:28,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T07:15:28,946 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=409, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/f9c69276fafd4f51801adc05eb470712 2024-11-17T07:15:28,951 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/594e39c67b434572a0de0633fe451201 is 50, key is test_row_0/B:col10/1731827725567/Put/seqid=0 2024-11-17T07:15:28,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742283_1459 (size=12301) 2024-11-17T07:15:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T07:15:29,355 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/594e39c67b434572a0de0633fe451201 2024-11-17T07:15:29,360 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/3b59841aa47c4f239696793977e2e521 is 50, key is test_row_0/C:col10/1731827725567/Put/seqid=0 2024-11-17T07:15:29,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742284_1460 (size=12301) 2024-11-17T07:15:29,764 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/3b59841aa47c4f239696793977e2e521 2024-11-17T07:15:29,767 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/A/f9c69276fafd4f51801adc05eb470712 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/f9c69276fafd4f51801adc05eb470712 2024-11-17T07:15:29,770 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/f9c69276fafd4f51801adc05eb470712, entries=150, sequenceid=409, filesize=30.5 K 2024-11-17T07:15:29,771 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/B/594e39c67b434572a0de0633fe451201 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/594e39c67b434572a0de0633fe451201 2024-11-17T07:15:29,773 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/594e39c67b434572a0de0633fe451201, entries=150, sequenceid=409, filesize=12.0 K 2024-11-17T07:15:29,774 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/.tmp/C/3b59841aa47c4f239696793977e2e521 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3b59841aa47c4f239696793977e2e521 2024-11-17T07:15:29,776 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3b59841aa47c4f239696793977e2e521, entries=150, sequenceid=409, filesize=12.0 K 2024-11-17T07:15:29,777 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d0c00ab4c3958615a6e7b967414841cf in 1648ms, sequenceid=409, compaction requested=true 2024-11-17T07:15:29,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/444d687d4452408e87f6b1a309c4d3fd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/b99a22160061444cb18300fb048db293, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/742385263d7345f5858cba608b7d4b17, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/ed4c0b38642043899b1875e286f32938, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/1da8549fe0d54ca0b87be9dcdbe023af, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5555ee63799a46b0a54607f9ac8d1df6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/82d714f654d4478d9619c504884fd8eb, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/43047641aadb4f73b0ef3459bd4bdbec, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/da0fa3b02fde4528b13cfa0f57f40dd5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/1361bde36c424f889f39ae2966fe36f3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/72e34d99cea64d21bc01ff6b56ad6676, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/29ac3d3069c449cb9ac8a0a9cbf68bb1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a56558b2d1a44f0a9769a9b149978308, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/0787f6d2047049ae8ee25d3876c2b8e6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/9b6055982542489ba0d36faaf247c0d4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5429a402a3aa4d11a36d116458dc7108, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/c750205c6db34319bd8a7f8fc835b92c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/8647fb43c0c14c5bac0ae19ea7436a7d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/d2799e192c284d36b39e19e1215d0f18, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a6687d70f4cd4dac91287be31e33eed5] to archive 2024-11-17T07:15:29,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:15:29,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/444d687d4452408e87f6b1a309c4d3fd to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/444d687d4452408e87f6b1a309c4d3fd 2024-11-17T07:15:29,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/b99a22160061444cb18300fb048db293 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/b99a22160061444cb18300fb048db293 2024-11-17T07:15:29,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/742385263d7345f5858cba608b7d4b17 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/742385263d7345f5858cba608b7d4b17 2024-11-17T07:15:29,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/ed4c0b38642043899b1875e286f32938 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/ed4c0b38642043899b1875e286f32938 2024-11-17T07:15:29,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/1da8549fe0d54ca0b87be9dcdbe023af to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/1da8549fe0d54ca0b87be9dcdbe023af 2024-11-17T07:15:29,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5555ee63799a46b0a54607f9ac8d1df6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5555ee63799a46b0a54607f9ac8d1df6 2024-11-17T07:15:29,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/82d714f654d4478d9619c504884fd8eb to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/82d714f654d4478d9619c504884fd8eb 2024-11-17T07:15:29,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/43047641aadb4f73b0ef3459bd4bdbec to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/43047641aadb4f73b0ef3459bd4bdbec 2024-11-17T07:15:29,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/da0fa3b02fde4528b13cfa0f57f40dd5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/da0fa3b02fde4528b13cfa0f57f40dd5 2024-11-17T07:15:29,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/1361bde36c424f889f39ae2966fe36f3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/1361bde36c424f889f39ae2966fe36f3 2024-11-17T07:15:29,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/72e34d99cea64d21bc01ff6b56ad6676 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/72e34d99cea64d21bc01ff6b56ad6676 2024-11-17T07:15:29,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/29ac3d3069c449cb9ac8a0a9cbf68bb1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/29ac3d3069c449cb9ac8a0a9cbf68bb1 2024-11-17T07:15:29,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a56558b2d1a44f0a9769a9b149978308 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a56558b2d1a44f0a9769a9b149978308 2024-11-17T07:15:29,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/0787f6d2047049ae8ee25d3876c2b8e6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/0787f6d2047049ae8ee25d3876c2b8e6 2024-11-17T07:15:29,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/9b6055982542489ba0d36faaf247c0d4 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/9b6055982542489ba0d36faaf247c0d4 2024-11-17T07:15:29,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5429a402a3aa4d11a36d116458dc7108 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/5429a402a3aa4d11a36d116458dc7108 2024-11-17T07:15:29,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/c750205c6db34319bd8a7f8fc835b92c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/c750205c6db34319bd8a7f8fc835b92c 2024-11-17T07:15:29,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/8647fb43c0c14c5bac0ae19ea7436a7d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/8647fb43c0c14c5bac0ae19ea7436a7d 2024-11-17T07:15:29,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/d2799e192c284d36b39e19e1215d0f18 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/d2799e192c284d36b39e19e1215d0f18 2024-11-17T07:15:29,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a6687d70f4cd4dac91287be31e33eed5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/a6687d70f4cd4dac91287be31e33eed5 2024-11-17T07:15:29,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/c8b6b9f6b74e4e43b504812dd2514fb0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7f800554ca7c43af81956222075077b0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/da1b4f153e2a41debe9cc57ccd293abe, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a3dc7fb42b3e42e5b9cb84a5438213c0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/194c5c922655462ebf66bdffc000ee71, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/049c09ac3a4040a68634c12ca6416e5d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/f8a55aa1935146fd8e5ccb917a9cb713, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a2fb45b9c1164c03923811f3612d461c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/f19ddb9e87f14f17b710757ae2b38b54, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7142dedf087946e7b8f6fb84bd460d45, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/0ea84767dfd448218e4ae9ba61358b30, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/0647991d4e6446bfaf454330e5ef5a9d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7d3acb90ba86494aabcb3531f9b64384, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/be15eedf3cb34b4bad45c242e6d51205, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a0b59bb115b7451981b50516c247a197, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/417a3307d0014c4c86eef3141d217548, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b89410487e72436e88dc6cf77327698f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/9b593991d6404552b82893f52b97d538, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/469baded619845fc96a496d2554c2eb3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b71e7c449c6e4cf4b0cf95fb475b03f5] to archive 2024-11-17T07:15:29,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:15:29,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/c8b6b9f6b74e4e43b504812dd2514fb0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/c8b6b9f6b74e4e43b504812dd2514fb0 2024-11-17T07:15:29,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7f800554ca7c43af81956222075077b0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7f800554ca7c43af81956222075077b0 2024-11-17T07:15:29,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/da1b4f153e2a41debe9cc57ccd293abe to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/da1b4f153e2a41debe9cc57ccd293abe 2024-11-17T07:15:29,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a3dc7fb42b3e42e5b9cb84a5438213c0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a3dc7fb42b3e42e5b9cb84a5438213c0 2024-11-17T07:15:29,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/194c5c922655462ebf66bdffc000ee71 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/194c5c922655462ebf66bdffc000ee71 2024-11-17T07:15:29,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/049c09ac3a4040a68634c12ca6416e5d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/049c09ac3a4040a68634c12ca6416e5d 2024-11-17T07:15:29,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/f8a55aa1935146fd8e5ccb917a9cb713 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/f8a55aa1935146fd8e5ccb917a9cb713 2024-11-17T07:15:29,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a2fb45b9c1164c03923811f3612d461c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a2fb45b9c1164c03923811f3612d461c 2024-11-17T07:15:29,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/f19ddb9e87f14f17b710757ae2b38b54 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/f19ddb9e87f14f17b710757ae2b38b54 2024-11-17T07:15:29,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7142dedf087946e7b8f6fb84bd460d45 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7142dedf087946e7b8f6fb84bd460d45 2024-11-17T07:15:29,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/0ea84767dfd448218e4ae9ba61358b30 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/0ea84767dfd448218e4ae9ba61358b30 2024-11-17T07:15:29,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/0647991d4e6446bfaf454330e5ef5a9d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/0647991d4e6446bfaf454330e5ef5a9d 2024-11-17T07:15:29,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7d3acb90ba86494aabcb3531f9b64384 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/7d3acb90ba86494aabcb3531f9b64384 2024-11-17T07:15:29,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/be15eedf3cb34b4bad45c242e6d51205 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/be15eedf3cb34b4bad45c242e6d51205 2024-11-17T07:15:29,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a0b59bb115b7451981b50516c247a197 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/a0b59bb115b7451981b50516c247a197 2024-11-17T07:15:29,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/417a3307d0014c4c86eef3141d217548 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/417a3307d0014c4c86eef3141d217548 2024-11-17T07:15:29,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b89410487e72436e88dc6cf77327698f to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b89410487e72436e88dc6cf77327698f 2024-11-17T07:15:29,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/9b593991d6404552b82893f52b97d538 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/9b593991d6404552b82893f52b97d538 2024-11-17T07:15:29,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/469baded619845fc96a496d2554c2eb3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/469baded619845fc96a496d2554c2eb3 2024-11-17T07:15:29,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b71e7c449c6e4cf4b0cf95fb475b03f5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b71e7c449c6e4cf4b0cf95fb475b03f5 2024-11-17T07:15:29,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/44eb0517d2e341a7bce7188774c1aa34, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/e2798f53ea5049ff812a4b15cd4635ab, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/a7a49c5ca33c4060ae30e82215ba7503, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/59155d3b650c4850bcd481a791055f2d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3b50e12eb0534ffab65b2ed1a63ce2d9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/428935610ded49359ff9f2a05613a459, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/062e459681234604aac8c859169f2808, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/a29fcdb977544815b1d129b1362713d0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5ba39c79161a4915b87af676c2dcba11, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/13e390661aed4ea7a27c681047bbd8e0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/bea149b622154a539b3697354dcf0f76, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/9c04ebde36ff469e95a2031195041343, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5bd751fbf5c94a2fbe407b9591bd2a4e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/8ebde51a56e940b0a3447304fc1f3055, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/85d9e2cc98fb4f4a9cbd9001899f1173, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/e7effd6d2b4e4380af607f7f33d3e5fe, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5c98d41a2815474a8727ccf617145e70, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/f371f4e24eef45b5bde0c41a9646e7bc, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3bc55412ce994951ab1c610c78ae1356, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/d5a7676dab1741cda5048e889ada0e27] to archive 2024-11-17T07:15:29,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:15:29,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/44eb0517d2e341a7bce7188774c1aa34 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/44eb0517d2e341a7bce7188774c1aa34 2024-11-17T07:15:29,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/e2798f53ea5049ff812a4b15cd4635ab to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/e2798f53ea5049ff812a4b15cd4635ab 2024-11-17T07:15:29,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/a7a49c5ca33c4060ae30e82215ba7503 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/a7a49c5ca33c4060ae30e82215ba7503 2024-11-17T07:15:29,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/59155d3b650c4850bcd481a791055f2d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/59155d3b650c4850bcd481a791055f2d 2024-11-17T07:15:29,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3b50e12eb0534ffab65b2ed1a63ce2d9 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3b50e12eb0534ffab65b2ed1a63ce2d9 2024-11-17T07:15:29,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/428935610ded49359ff9f2a05613a459 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/428935610ded49359ff9f2a05613a459 2024-11-17T07:15:29,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/062e459681234604aac8c859169f2808 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/062e459681234604aac8c859169f2808 2024-11-17T07:15:29,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/a29fcdb977544815b1d129b1362713d0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/a29fcdb977544815b1d129b1362713d0 2024-11-17T07:15:29,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5ba39c79161a4915b87af676c2dcba11 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5ba39c79161a4915b87af676c2dcba11 2024-11-17T07:15:29,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/13e390661aed4ea7a27c681047bbd8e0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/13e390661aed4ea7a27c681047bbd8e0 2024-11-17T07:15:29,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/bea149b622154a539b3697354dcf0f76 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/bea149b622154a539b3697354dcf0f76 2024-11-17T07:15:29,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/9c04ebde36ff469e95a2031195041343 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/9c04ebde36ff469e95a2031195041343 2024-11-17T07:15:29,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5bd751fbf5c94a2fbe407b9591bd2a4e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5bd751fbf5c94a2fbe407b9591bd2a4e 2024-11-17T07:15:29,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/8ebde51a56e940b0a3447304fc1f3055 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/8ebde51a56e940b0a3447304fc1f3055 2024-11-17T07:15:29,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/85d9e2cc98fb4f4a9cbd9001899f1173 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/85d9e2cc98fb4f4a9cbd9001899f1173 2024-11-17T07:15:29,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/e7effd6d2b4e4380af607f7f33d3e5fe to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/e7effd6d2b4e4380af607f7f33d3e5fe 2024-11-17T07:15:29,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5c98d41a2815474a8727ccf617145e70 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/5c98d41a2815474a8727ccf617145e70 2024-11-17T07:15:29,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/f371f4e24eef45b5bde0c41a9646e7bc to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/f371f4e24eef45b5bde0c41a9646e7bc 2024-11-17T07:15:29,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3bc55412ce994951ab1c610c78ae1356 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3bc55412ce994951ab1c610c78ae1356 2024-11-17T07:15:29,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/d5a7676dab1741cda5048e889ada0e27 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/d5a7676dab1741cda5048e889ada0e27 2024-11-17T07:15:29,829 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/recovered.edits/412.seqid, newMaxSeqId=412, maxSeqId=4 2024-11-17T07:15:29,830 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. 2024-11-17T07:15:29,830 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for d0c00ab4c3958615a6e7b967414841cf: Waiting for close lock at 1731827728129Running coprocessor pre-close hooks at 1731827728129Disabling compacts and flushes for region at 1731827728129Disabling writes for close at 1731827728129Obtaining lock to block concurrent updates at 1731827728129Preparing flush snapshotting stores in d0c00ab4c3958615a6e7b967414841cf at 1731827728129Finished memstore snapshotting TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731827728129Flushing stores of TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf. at 1731827728130 (+1 ms)Flushing d0c00ab4c3958615a6e7b967414841cf/A: creating writer at 1731827728130Flushing mob file d0c00ab4c3958615a6e7b967414841cf/A: appending metadata at 1731827728134 (+4 ms)Flushing mob file d0c00ab4c3958615a6e7b967414841cf/A: closing flushed file at 1731827728134Flushing d0c00ab4c3958615a6e7b967414841cf/A: appending metadata at 1731827728542 (+408 ms)Flushing d0c00ab4c3958615a6e7b967414841cf/A: closing flushed file at 1731827728542Flushing d0c00ab4c3958615a6e7b967414841cf/B: creating writer at 1731827728949 (+407 ms)Flushing d0c00ab4c3958615a6e7b967414841cf/B: appending metadata at 1731827728951 (+2 ms)Flushing d0c00ab4c3958615a6e7b967414841cf/B: closing flushed file at 1731827728951Flushing d0c00ab4c3958615a6e7b967414841cf/C: creating writer at 1731827729358 (+407 ms)Flushing d0c00ab4c3958615a6e7b967414841cf/C: appending metadata at 1731827729360 (+2 ms)Flushing d0c00ab4c3958615a6e7b967414841cf/C: closing flushed file at 1731827729360Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ef25668: reopening flushed file at 1731827729767 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d8d5480: reopening flushed file at 1731827729770 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fcf43f2: reopening flushed file at 1731827729773 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for d0c00ab4c3958615a6e7b967414841cf in 1648ms, sequenceid=409, compaction requested=true at 1731827729777 (+4 ms)Writing region close event to WAL at 1731827729827 (+50 ms)Running coprocessor post-close hooks at 1731827729829 (+2 ms)Closed at 1731827729830 (+1 ms) 2024-11-17T07:15:29,831 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:29,832 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=d0c00ab4c3958615a6e7b967414841cf, regionState=CLOSED 2024-11-17T07:15:29,833 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:15:29,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-17T07:15:29,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure d0c00ab4c3958615a6e7b967414841cf, server=1dd62e24b303,36419,1731827614816 in 1.8560 sec 2024-11-17T07:15:29,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-17T07:15:29,836 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d0c00ab4c3958615a6e7b967414841cf, UNASSIGN in 1.8610 sec 2024-11-17T07:15:29,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-17T07:15:29,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8630 sec 2024-11-17T07:15:29,839 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827729838"}]},"ts":"1731827729838"} 2024-11-17T07:15:29,839 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T07:15:29,839 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T07:15:29,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8710 sec 2024-11-17T07:15:30,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-17T07:15:30,117 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:30,118 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T07:15:30,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:30,119 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:30,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T07:15:30,120 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:30,122 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,124 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/recovered.edits] 2024-11-17T07:15:30,126 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/47515bc88bea448ea66243571b1b7682 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/47515bc88bea448ea66243571b1b7682 2024-11-17T07:15:30,127 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/d010a282654c4ce9a43be2e88cde98bc to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/d010a282654c4ce9a43be2e88cde98bc 2024-11-17T07:15:30,128 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/f9c69276fafd4f51801adc05eb470712 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/A/f9c69276fafd4f51801adc05eb470712 2024-11-17T07:15:30,130 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/594e39c67b434572a0de0633fe451201 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/594e39c67b434572a0de0633fe451201 2024-11-17T07:15:30,131 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b4ef1c9fa84d4169b0e7f934014f1347 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/b4ef1c9fa84d4169b0e7f934014f1347 2024-11-17T07:15:30,131 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/cf70546c8c524a25a2bd9279a952a459 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/B/cf70546c8c524a25a2bd9279a952a459 2024-11-17T07:15:30,133 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3b59841aa47c4f239696793977e2e521 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/3b59841aa47c4f239696793977e2e521 2024-11-17T07:15:30,134 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/44425bba8cde4900a4ce8a9e0aa79712 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/44425bba8cde4900a4ce8a9e0aa79712 2024-11-17T07:15:30,135 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/ec3a4ebd872d4ece95245154012391e8 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/C/ec3a4ebd872d4ece95245154012391e8 2024-11-17T07:15:30,137 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/recovered.edits/412.seqid to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf/recovered.edits/412.seqid 2024-11-17T07:15:30,137 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,137 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T07:15:30,137 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T07:15:30,138 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T07:15:30,140 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170849f50adc3846339b562bf394962d2c_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170849f50adc3846339b562bf394962d2c_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,141 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170dabe548ca8d4a78bbe69c0fa81a25b1_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170dabe548ca8d4a78bbe69c0fa81a25b1_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,142 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111723c479d5aff34414ac15fd2f79267de9_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111723c479d5aff34414ac15fd2f79267de9_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,143 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174243b3ddc29c4da7852d32d4834f94f6_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174243b3ddc29c4da7852d32d4834f94f6_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,144 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174c9b2b73ea1042c6bbbcc47ec9ba3fa6_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174c9b2b73ea1042c6bbbcc47ec9ba3fa6_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,144 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111757ede32796984681ba5543b43bc3bdc3_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111757ede32796984681ba5543b43bc3bdc3_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,145 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111769a5623a61894dffab49d93578f753bb_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111769a5623a61894dffab49d93578f753bb_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,146 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176dea00ba287847ea903c65dc6d006cd5_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176dea00ba287847ea903c65dc6d006cd5_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,147 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117818dbc0478604052bd5a43e8891a42f4_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117818dbc0478604052bd5a43e8891a42f4_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,148 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178316a29a2fb847a982b099c8421c1211_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411178316a29a2fb847a982b099c8421c1211_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,148 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117934a55a7ee014dc69b6c8d3b4db8cec2_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117934a55a7ee014dc69b6c8d3b4db8cec2_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,149 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a169cc591246442ab6824f200c65ba7a_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117a169cc591246442ab6824f200c65ba7a_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,150 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c82471c3b68747ea915fb981d97b7053_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117c82471c3b68747ea915fb981d97b7053_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,151 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e5b95eb36c9e4b91bce6878ffb499dcd_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e5b95eb36c9e4b91bce6878ffb499dcd_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,151 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117edd006918c124a148bcea9cc90032eb5_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117edd006918c124a148bcea9cc90032eb5_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,152 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eec3bd2ce9b847f3bc761be650b20daf_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117eec3bd2ce9b847f3bc761be650b20daf_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,153 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ff728ed0da724cfe9fe630abf3f98dbc_d0c00ab4c3958615a6e7b967414841cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117ff728ed0da724cfe9fe630abf3f98dbc_d0c00ab4c3958615a6e7b967414841cf 2024-11-17T07:15:30,154 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T07:15:30,155 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:30,157 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T07:15:30,159 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T07:15:30,159 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:30,159 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T07:15:30,160 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731827730159"}]},"ts":"9223372036854775807"} 2024-11-17T07:15:30,161 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T07:15:30,161 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d0c00ab4c3958615a6e7b967414841cf, NAME => 'TestAcidGuarantees,,1731827701606.d0c00ab4c3958615a6e7b967414841cf.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T07:15:30,161 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T07:15:30,161 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731827730161"}]},"ts":"9223372036854775807"} 2024-11-17T07:15:30,162 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T07:15:30,163 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:30,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-17T07:15:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-17T07:15:30,227 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T07:15:30,227 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:30,236 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 249) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=396 (was 372) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1085 (was 1182) 2024-11-17T07:15:30,244 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=396, ProcessCount=11, AvailableMemoryMB=1085 2024-11-17T07:15:30,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T07:15:30,245 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T07:15:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:30,247 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T07:15:30,247 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-17T07:15:30,247 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:30,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T07:15:30,248 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T07:15:30,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742285_1461 (size=1061) 2024-11-17T07:15:30,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T07:15:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T07:15:30,655 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650 2024-11-17T07:15:30,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742286_1462 (size=53) 2024-11-17T07:15:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T07:15:31,060 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:15:31,060 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing cab1e460ee74e5eada90871d0516a7f5, disabling compactions & flushes 2024-11-17T07:15:31,060 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:31,060 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:31,060 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. after waiting 0 ms 2024-11-17T07:15:31,060 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:31,060 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:31,060 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for cab1e460ee74e5eada90871d0516a7f5: Waiting for close lock at 1731827731060Disabling compacts and flushes for region at 1731827731060Disabling writes for close at 1731827731060Writing region close event to WAL at 1731827731060Closed at 1731827731060 2024-11-17T07:15:31,061 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T07:15:31,061 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731827731061"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731827731061"}]},"ts":"1731827731061"} 2024-11-17T07:15:31,062 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T07:15:31,063 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T07:15:31,063 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827731063"}]},"ts":"1731827731063"} 2024-11-17T07:15:31,064 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T07:15:31,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cab1e460ee74e5eada90871d0516a7f5, ASSIGN}] 2024-11-17T07:15:31,065 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cab1e460ee74e5eada90871d0516a7f5, ASSIGN 2024-11-17T07:15:31,066 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cab1e460ee74e5eada90871d0516a7f5, ASSIGN; state=OFFLINE, location=1dd62e24b303,36419,1731827614816; forceNewPlan=false, retain=false 2024-11-17T07:15:31,216 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=cab1e460ee74e5eada90871d0516a7f5, regionState=OPENING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:31,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cab1e460ee74e5eada90871d0516a7f5, ASSIGN because future has completed 2024-11-17T07:15:31,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:15:31,372 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:31,372 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} 2024-11-17T07:15:31,373 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:31,373 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:15:31,373 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:31,373 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:31,374 INFO [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:31,375 INFO [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:15:31,375 INFO [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cab1e460ee74e5eada90871d0516a7f5 columnFamilyName A 2024-11-17T07:15:31,375 DEBUG [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:31,376 INFO [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] regionserver.HStore(327): Store=cab1e460ee74e5eada90871d0516a7f5/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:15:31,376 INFO [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:31,377 INFO [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:15:31,377 INFO [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cab1e460ee74e5eada90871d0516a7f5 columnFamilyName B 2024-11-17T07:15:31,377 DEBUG [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:31,377 INFO [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] regionserver.HStore(327): Store=cab1e460ee74e5eada90871d0516a7f5/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:15:31,377 INFO [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:31,378 INFO [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:15:31,378 INFO [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cab1e460ee74e5eada90871d0516a7f5 columnFamilyName C 2024-11-17T07:15:31,378 DEBUG [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:31,378 INFO [StoreOpener-cab1e460ee74e5eada90871d0516a7f5-1 {}] regionserver.HStore(327): Store=cab1e460ee74e5eada90871d0516a7f5/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:15:31,378 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:31,378 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:31,379 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:31,379 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:31,380 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:31,380 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:31,381 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T07:15:31,382 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:31,383 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T07:15:31,383 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened cab1e460ee74e5eada90871d0516a7f5; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59807632, jitterRate=-0.10879683494567871}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T07:15:31,383 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:31,384 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for cab1e460ee74e5eada90871d0516a7f5: Running coprocessor pre-open hook at 1731827731373Writing region info on filesystem at 1731827731373Initializing all the Stores at 1731827731374 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827731374Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827731374Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827731374Cleaning up temporary data from old regions at 1731827731380 (+6 ms)Running coprocessor post-open hooks at 1731827731383 (+3 ms)Region opened successfully at 1731827731384 (+1 ms) 2024-11-17T07:15:31,385 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., pid=122, masterSystemTime=1731827731369 2024-11-17T07:15:31,386 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:31,386 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:31,386 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=cab1e460ee74e5eada90871d0516a7f5, regionState=OPEN, openSeqNum=2, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:31,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T07:15:31,388 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:15:31,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-17T07:15:31,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 in 170 msec 2024-11-17T07:15:31,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-17T07:15:31,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cab1e460ee74e5eada90871d0516a7f5, ASSIGN in 325 msec 2024-11-17T07:15:31,391 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T07:15:31,392 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827731391"}]},"ts":"1731827731391"} 2024-11-17T07:15:31,393 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T07:15:31,393 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T07:15:31,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-11-17T07:15:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-17T07:15:32,398 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:32,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a364c5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:32,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:32,400 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59abe20d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:32,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:32,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:32,401 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:32,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@682ac573, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,402 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:32,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:32,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:32,404 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T07:15:32,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 1dd62e24b303,36491,1731827613877 2024-11-17T07:15:32,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@61dfad0b 2024-11-17T07:15:32,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T07:15:32,405 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T07:15:32,406 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6516064e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,406 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:32,407 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:32,407 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108525dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:32,408 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:32,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@199b9d12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:32,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:32,410 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45579da8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:32,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:32,411 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:32,411 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526a4896, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,412 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:32,412 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:32,412 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22f58724, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:32,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:32,413 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:32,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f1f513f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,414 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,414 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:32,414 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:32,414 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:32,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:32,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:32,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c562208, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:32,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:32,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:32,415 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31131b6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:32,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:32,417 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e15db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:32,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:32,418 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:32,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31247154, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,419 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:32,419 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:32,419 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@879f458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:32,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:32,420 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:32,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a647334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:32,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:32,422 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b1dd2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:32,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:32,423 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50980, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29089fc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:32,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:32,424 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b383750, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:32,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:32,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:32,425 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:32,426 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69cf1eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:32,426 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:32,427 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@552261d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:32,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:32,428 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:32,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4981c1c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,429 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,429 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:15:32,429 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:15:32,429 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:15:32,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:15:32,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:15:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f5b60b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:15:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:15:32,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:32,430 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:15:32,431 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b12fc82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:15:32,431 DEBUG [Thread-2067 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:32,431 DEBUG [Thread-2078 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:32,431 DEBUG [Thread-2071 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:32,431 DEBUG [Thread-2082 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:32,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:32,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:32,432 DEBUG [Thread-2080 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:32,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:32,432 DEBUG [Thread-2065 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:32,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:32,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:32,432 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:32,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:32,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:32,433 DEBUG [Thread-2069 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:32,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:32,433 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56822, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:32,434 DEBUG [Thread-2063 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:32,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:32,434 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:32,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:32,434 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56850, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:32,435 DEBUG [Thread-2074 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:32,436 DEBUG [Thread-2076 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:15:32,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:32,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-17T07:15:32,436 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T07:15:32,437 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:15:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:15:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:15:32,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:32,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:15:32,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:15:32,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:32,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:15:32,438 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:32,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56906, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:32,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:15:32,440 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:32,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:32,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:15:32,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:15:32,441 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:15:32,442 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:15:32,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:15:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2] 2024-11-17T07:15:32,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:32,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:15:32,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:32,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:32,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:32,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:32,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:32,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:32,483 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/f45cd89b0011470d97df1fddc5c2a944 is 50, key is test_row_0/A:col10/1731827732452/Put/seqid=0 2024-11-17T07:15:32,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742287_1463 (size=12001) 2024-11-17T07:15:32,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827792538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827732464, max timestamp=1731827732546Num uniques 859; 2024-11-17T07:15:32,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:15:32,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827732464, max timestamp=1731827732546Num uniques 859; 2024-11-17T07:15:32,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827732464, max timestamp=1731827732546Num uniques 859; 2024-11-17T07:15:32,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:15:32,548 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:15:32,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T07:15:32,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827792578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827792579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,592 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827792581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,593 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827792591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T07:15:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:32,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:32,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:32,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:32,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827792657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,664 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827792697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827792698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,705 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827792698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827792698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T07:15:32,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:32,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:32,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:32,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:32,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:32,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T07:15:32,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827792867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/f45cd89b0011470d97df1fddc5c2a944 2024-11-17T07:15:32,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T07:15:32,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:32,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:32,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:32,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:32,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:32,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:32,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/8854e5a14dd54560ad33e982bfdca402 is 50, key is test_row_0/B:col10/1731827732452/Put/seqid=0 2024-11-17T07:15:32,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827792907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742288_1464 (size=12001) 2024-11-17T07:15:32,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827792917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827792917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:32,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827792917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:32,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:32,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:32,932 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T07:15:33,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T07:15:33,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:33,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:33,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:33,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T07:15:33,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:33,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827793187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:33,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T07:15:33,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:33,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:33,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:33,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:33,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827793217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:33,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827793237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:33,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827793237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:33,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:33,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827793237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:33,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/8854e5a14dd54560ad33e982bfdca402 2024-11-17T07:15:33,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/09fb35ad8d2c44da8999a41fff409383 is 50, key is test_row_0/C:col10/1731827732452/Put/seqid=0 2024-11-17T07:15:33,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742289_1465 (size=12001) 2024-11-17T07:15:33,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T07:15:33,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:33,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:33,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:33,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T07:15:33,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:33,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:33,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:33,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T07:15:33,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T07:15:33,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:33,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:33,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:33,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:33,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827793707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:33,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:33,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:33,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827793727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:33,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:33,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/09fb35ad8d2c44da8999a41fff409383 2024-11-17T07:15:33,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/f45cd89b0011470d97df1fddc5c2a944 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f45cd89b0011470d97df1fddc5c2a944 2024-11-17T07:15:33,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f45cd89b0011470d97df1fddc5c2a944, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T07:15:33,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/8854e5a14dd54560ad33e982bfdca402 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/8854e5a14dd54560ad33e982bfdca402 2024-11-17T07:15:33,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/8854e5a14dd54560ad33e982bfdca402, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T07:15:33,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/09fb35ad8d2c44da8999a41fff409383 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/09fb35ad8d2c44da8999a41fff409383 2024-11-17T07:15:33,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827793747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:33,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:33,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:33,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827793747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:33,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827793747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:33,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:33,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/09fb35ad8d2c44da8999a41fff409383, entries=150, sequenceid=12, filesize=11.7 K 2024-11-17T07:15:33,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cab1e460ee74e5eada90871d0516a7f5 in 1308ms, sequenceid=12, compaction requested=false 2024-11-17T07:15:33,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:33,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-17T07:15:33,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:33,820 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-17T07:15:33,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:33,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:33,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:33,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:33,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:33,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/6d3bb7d6e48743139e5f37fefeb19477 is 50, key is test_row_0/A:col10/1731827732590/Put/seqid=0 2024-11-17T07:15:33,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742290_1466 (size=12001) 2024-11-17T07:15:34,229 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/6d3bb7d6e48743139e5f37fefeb19477 2024-11-17T07:15:34,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/c1c1289beb634ba3af2a318a447f4e2d is 50, key is test_row_0/B:col10/1731827732590/Put/seqid=0 2024-11-17T07:15:34,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742291_1467 (size=12001) 2024-11-17T07:15:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T07:15:34,641 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/c1c1289beb634ba3af2a318a447f4e2d 2024-11-17T07:15:34,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/0a45d419bbcd4a94bf8f32a8285fa118 is 50, key is test_row_0/C:col10/1731827732590/Put/seqid=0 2024-11-17T07:15:34,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742292_1468 (size=12001) 2024-11-17T07:15:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:34,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:34,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827794767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:34,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827794767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:34,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827794768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:34,771 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:34,771 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:34,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827794769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:34,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827794777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:34,784 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,785 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:34,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827794877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:34,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827794887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:34,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:35,057 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/0a45d419bbcd4a94bf8f32a8285fa118 2024-11-17T07:15:35,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/6d3bb7d6e48743139e5f37fefeb19477 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6d3bb7d6e48743139e5f37fefeb19477 2024-11-17T07:15:35,063 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6d3bb7d6e48743139e5f37fefeb19477, entries=150, sequenceid=47, filesize=11.7 K 2024-11-17T07:15:35,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/c1c1289beb634ba3af2a318a447f4e2d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c1c1289beb634ba3af2a318a447f4e2d 2024-11-17T07:15:35,067 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c1c1289beb634ba3af2a318a447f4e2d, entries=150, sequenceid=47, filesize=11.7 K 2024-11-17T07:15:35,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/0a45d419bbcd4a94bf8f32a8285fa118 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/0a45d419bbcd4a94bf8f32a8285fa118 2024-11-17T07:15:35,071 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/0a45d419bbcd4a94bf8f32a8285fa118, entries=150, sequenceid=47, filesize=11.7 K 2024-11-17T07:15:35,072 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for cab1e460ee74e5eada90871d0516a7f5 in 1253ms, sequenceid=47, compaction requested=false 2024-11-17T07:15:35,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:35,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:35,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-17T07:15:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-17T07:15:35,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-17T07:15:35,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6330 sec 2024-11-17T07:15:35,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.6410 sec 2024-11-17T07:15:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:35,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T07:15:35,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:35,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:35,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:35,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:35,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:35,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:35,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/30cfd9c3ed7c40baa74aa4e4477387f4 is 50, key is test_row_0/A:col10/1731827735098/Put/seqid=0 2024-11-17T07:15:35,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742293_1469 (size=14341) 2024-11-17T07:15:35,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/30cfd9c3ed7c40baa74aa4e4477387f4 2024-11-17T07:15:35,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/d1198d2d6c84429cbb22006e31fc584e is 50, key is test_row_0/B:col10/1731827735098/Put/seqid=0 2024-11-17T07:15:35,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742294_1470 (size=12001) 2024-11-17T07:15:35,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827795163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:35,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:35,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827795164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:35,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:35,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827795277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,287 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:35,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:35,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827795287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:35,351 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T07:15:35,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827795497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:35,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827795507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:35,523 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/d1198d2d6c84429cbb22006e31fc584e 2024-11-17T07:15:35,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/32793abb413848ee87e3348b909aaf3e is 50, key is test_row_0/C:col10/1731827735098/Put/seqid=0 2024-11-17T07:15:35,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742295_1471 (size=12001) 2024-11-17T07:15:35,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827795817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:35,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827795817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:35,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:35,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/32793abb413848ee87e3348b909aaf3e 2024-11-17T07:15:35,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/30cfd9c3ed7c40baa74aa4e4477387f4 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/30cfd9c3ed7c40baa74aa4e4477387f4 2024-11-17T07:15:35,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/30cfd9c3ed7c40baa74aa4e4477387f4, entries=200, sequenceid=59, filesize=14.0 K 2024-11-17T07:15:35,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/d1198d2d6c84429cbb22006e31fc584e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/d1198d2d6c84429cbb22006e31fc584e 2024-11-17T07:15:35,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/d1198d2d6c84429cbb22006e31fc584e, entries=150, sequenceid=59, filesize=11.7 K 2024-11-17T07:15:35,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/32793abb413848ee87e3348b909aaf3e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/32793abb413848ee87e3348b909aaf3e 2024-11-17T07:15:35,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/32793abb413848ee87e3348b909aaf3e, entries=150, sequenceid=59, filesize=11.7 K 2024-11-17T07:15:35,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cab1e460ee74e5eada90871d0516a7f5 in 850ms, sequenceid=59, compaction requested=true 2024-11-17T07:15:35,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:35,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:35,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:35,949 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:35,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:35,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:35,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:35,950 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:35,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:35,950 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:35,950 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/A is initiating minor compaction (all files) 2024-11-17T07:15:35,951 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/A in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:35,951 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f45cd89b0011470d97df1fddc5c2a944, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6d3bb7d6e48743139e5f37fefeb19477, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/30cfd9c3ed7c40baa74aa4e4477387f4] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=37.4 K 2024-11-17T07:15:35,951 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:35,951 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting f45cd89b0011470d97df1fddc5c2a944, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731827732443 2024-11-17T07:15:35,951 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/B is initiating minor compaction (all files) 2024-11-17T07:15:35,951 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/B in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:35,951 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/8854e5a14dd54560ad33e982bfdca402, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c1c1289beb634ba3af2a318a447f4e2d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/d1198d2d6c84429cbb22006e31fc584e] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=35.2 K 2024-11-17T07:15:35,951 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d3bb7d6e48743139e5f37fefeb19477, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731827732577 2024-11-17T07:15:35,952 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 8854e5a14dd54560ad33e982bfdca402, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731827732443 2024-11-17T07:15:35,952 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30cfd9c3ed7c40baa74aa4e4477387f4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731827734745 2024-11-17T07:15:35,952 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting c1c1289beb634ba3af2a318a447f4e2d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731827732577 2024-11-17T07:15:35,952 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting d1198d2d6c84429cbb22006e31fc584e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731827734745 2024-11-17T07:15:35,959 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#A#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:35,960 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#B#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:35,960 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/f41c4c052c474c59be0197859376f5a9 is 50, key is test_row_0/A:col10/1731827735098/Put/seqid=0 2024-11-17T07:15:35,960 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/db60da51e5db4d7a9a8f0471e6f95c18 is 50, key is test_row_0/B:col10/1731827735098/Put/seqid=0 2024-11-17T07:15:35,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742296_1472 (size=12104) 2024-11-17T07:15:35,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742297_1473 (size=12104) 2024-11-17T07:15:35,979 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/db60da51e5db4d7a9a8f0471e6f95c18 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/db60da51e5db4d7a9a8f0471e6f95c18 2024-11-17T07:15:35,983 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/B of cab1e460ee74e5eada90871d0516a7f5 into db60da51e5db4d7a9a8f0471e6f95c18(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:35,983 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:35,983 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/B, priority=13, startTime=1731827735949; duration=0sec 2024-11-17T07:15:35,983 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:35,983 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:B 2024-11-17T07:15:35,983 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:35,984 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:35,984 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/C is initiating minor compaction (all files) 2024-11-17T07:15:35,984 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/C in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:35,984 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/09fb35ad8d2c44da8999a41fff409383, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/0a45d419bbcd4a94bf8f32a8285fa118, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/32793abb413848ee87e3348b909aaf3e] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=35.2 K 2024-11-17T07:15:35,985 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 09fb35ad8d2c44da8999a41fff409383, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731827732443 2024-11-17T07:15:35,985 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a45d419bbcd4a94bf8f32a8285fa118, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731827732577 2024-11-17T07:15:35,986 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 32793abb413848ee87e3348b909aaf3e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731827734745 2024-11-17T07:15:35,993 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:35,994 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/331deeae69a244d48fd93c38f3903165 is 50, key is test_row_0/C:col10/1731827735098/Put/seqid=0 2024-11-17T07:15:35,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742298_1474 (size=12104) 2024-11-17T07:15:36,006 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/331deeae69a244d48fd93c38f3903165 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/331deeae69a244d48fd93c38f3903165 2024-11-17T07:15:36,009 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/C of cab1e460ee74e5eada90871d0516a7f5 into 331deeae69a244d48fd93c38f3903165(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:36,010 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:36,010 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/C, priority=13, startTime=1731827735950; duration=0sec 2024-11-17T07:15:36,010 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:36,010 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:C 2024-11-17T07:15:36,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:36,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-17T07:15:36,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827735145, max timestamp=1731827736342Num uniques 859; 2024-11-17T07:15:36,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827735145, max timestamp=1731827736342Num uniques 859; 2024-11-17T07:15:36,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:36,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:36,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827735145, max timestamp=1731827736342Num uniques 859; 2024-11-17T07:15:36,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:36,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:36,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/fcb2d11fbd064ab4af599923dc3e7d48 is 50, key is test_row_0/A:col10/1731827736344/Put/seqid=0 2024-11-17T07:15:36,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742299_1475 (size=16681) 2024-11-17T07:15:36,375 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/f41c4c052c474c59be0197859376f5a9 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f41c4c052c474c59be0197859376f5a9 2024-11-17T07:15:36,381 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/A of cab1e460ee74e5eada90871d0516a7f5 into f41c4c052c474c59be0197859376f5a9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:36,381 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:36,381 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/A, priority=13, startTime=1731827735949; duration=0sec 2024-11-17T07:15:36,381 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:36,381 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:A 2024-11-17T07:15:36,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827736370, max timestamp=1731827736414Num uniques 859; 2024-11-17T07:15:36,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:15:36,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827736370, max timestamp=1731827736414Num uniques 859; 2024-11-17T07:15:36,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:15:36,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731827736370, max timestamp=1731827736414Num uniques 859; 2024-11-17T07:15:36,419 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:15:36,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731827736475, max timestamp=1731827736498Num uniques 1068; 2024-11-17T07:15:36,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731827736475, max timestamp=1731827736498Num uniques 1068; 2024-11-17T07:15:36,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-17T07:15:36,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-17T07:15:36,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731827736475, max timestamp=1731827736498Num uniques 1068; 2024-11-17T07:15:36,503 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-17T07:15:36,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731827736558, max timestamp=1731827736584Num uniques 1027; 2024-11-17T07:15:36,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-17T07:15:36,588 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731827736558, max timestamp=1731827736584Num uniques 1027; 2024-11-17T07:15:36,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-17T07:15:36,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731827736558, max timestamp=1731827736584Num uniques 1027; 2024-11-17T07:15:36,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-17T07:15:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-17T07:15:36,607 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:36,607 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-17T07:15:36,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T07:15:36,609 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:36,610 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:36,610 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:36,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731827736610, max timestamp=1731827736660Num uniques 1086; 2024-11-17T07:15:36,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-17T07:15:36,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731827736610, max timestamp=1731827736660Num uniques 1086; 2024-11-17T07:15:36,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731827736610, max timestamp=1731827736660Num uniques 1086; 2024-11-17T07:15:36,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-17T07:15:36,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-17T07:15:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T07:15:36,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731827736692, max timestamp=1731827736740Num uniques 1145; 2024-11-17T07:15:36,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731827736692, max timestamp=1731827736740Num uniques 1145; 2024-11-17T07:15:36,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731827736692, max timestamp=1731827736740Num uniques 1145; 2024-11-17T07:15:36,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-17T07:15:36,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-17T07:15:36,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-17T07:15:36,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/fcb2d11fbd064ab4af599923dc3e7d48 2024-11-17T07:15:36,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T07:15:36,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:36,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:36,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:36,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:36,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:36,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/a1e49d8285794e219fa5bfc8da1c470e is 50, key is test_row_0/B:col10/1731827736344/Put/seqid=0 2024-11-17T07:15:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:36,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742300_1476 (size=12001) 2024-11-17T07:15:36,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/a1e49d8285794e219fa5bfc8da1c470e 2024-11-17T07:15:36,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/eda253111c814b0e96189a4926063b7d is 50, key is test_row_0/C:col10/1731827736344/Put/seqid=0 2024-11-17T07:15:36,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742301_1477 (size=12001) 2024-11-17T07:15:36,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/eda253111c814b0e96189a4926063b7d 2024-11-17T07:15:36,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/fcb2d11fbd064ab4af599923dc3e7d48 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/fcb2d11fbd064ab4af599923dc3e7d48 2024-11-17T07:15:36,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/fcb2d11fbd064ab4af599923dc3e7d48, entries=250, sequenceid=83, filesize=16.3 K 2024-11-17T07:15:36,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/a1e49d8285794e219fa5bfc8da1c470e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/a1e49d8285794e219fa5bfc8da1c470e 2024-11-17T07:15:36,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731827736779, max timestamp=1731827736816Num uniques 1104; 2024-11-17T07:15:36,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-11-17T07:15:36,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731827736779, max timestamp=1731827736816Num uniques 1104; 2024-11-17T07:15:36,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-11-17T07:15:36,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=304, cellSize=13920, totalHeapSize=31856, min timestamp=1731827736779, max timestamp=1731827736816Num uniques 1104; 2024-11-17T07:15:36,822 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=13920, suffix heap size=142784,new segment heap size=31856 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=304 2024-11-17T07:15:36,826 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/a1e49d8285794e219fa5bfc8da1c470e, entries=150, sequenceid=83, filesize=11.7 K 2024-11-17T07:15:36,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/eda253111c814b0e96189a4926063b7d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/eda253111c814b0e96189a4926063b7d 2024-11-17T07:15:36,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/eda253111c814b0e96189a4926063b7d, entries=150, sequenceid=83, filesize=11.7 K 2024-11-17T07:15:36,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=53.67 KB/54960 for cab1e460ee74e5eada90871d0516a7f5 in 487ms, sequenceid=83, compaction requested=false 2024-11-17T07:15:36,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:36,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=60.38 KB heapSize=152.25 KB 2024-11-17T07:15:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:36,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:36,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/ec6e6e2c112545c89f862a168b1daf32 is 50, key is test_row_0/A:col10/1731827736831/Put/seqid=0 2024-11-17T07:15:36,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742302_1478 (size=16931) 2024-11-17T07:15:36,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/ec6e6e2c112545c89f862a168b1daf32 2024-11-17T07:15:36,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/d43dc67744fa40ab942d8496016fc8fd is 50, key is test_row_0/B:col10/1731827736831/Put/seqid=0 2024-11-17T07:15:36,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742303_1479 (size=12151) 2024-11-17T07:15:36,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/d43dc67744fa40ab942d8496016fc8fd 2024-11-17T07:15:36,885 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/563030a4d83844709472fdf56f64f53d is 50, key is test_row_0/C:col10/1731827736831/Put/seqid=0 2024-11-17T07:15:36,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742304_1480 (size=12151) 2024-11-17T07:15:36,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/563030a4d83844709472fdf56f64f53d 2024-11-17T07:15:36,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/ec6e6e2c112545c89f862a168b1daf32 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ec6e6e2c112545c89f862a168b1daf32 2024-11-17T07:15:36,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:36,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827796878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827796878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:36,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827796881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:36,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:36,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:36,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:36,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:36,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:36,900 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:36,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731827736872, max timestamp=1731827736894Num uniques 859; 2024-11-17T07:15:36,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:15:36,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731827736872, max timestamp=1731827736894Num uniques 859; 2024-11-17T07:15:36,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:15:36,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:36,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:36,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:36,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731827736872, max timestamp=1731827736894Num uniques 859; 2024-11-17T07:15:36,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-17T07:15:36,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ec6e6e2c112545c89f862a168b1daf32, entries=250, sequenceid=193, filesize=16.5 K 2024-11-17T07:15:36,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/d43dc67744fa40ab942d8496016fc8fd as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/d43dc67744fa40ab942d8496016fc8fd 2024-11-17T07:15:36,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/d43dc67744fa40ab942d8496016fc8fd, entries=150, sequenceid=193, filesize=11.9 K 2024-11-17T07:15:36,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/563030a4d83844709472fdf56f64f53d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/563030a4d83844709472fdf56f64f53d 2024-11-17T07:15:36,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T07:15:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:36,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:36,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:36,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:36,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/563030a4d83844709472fdf56f64f53d, entries=150, sequenceid=193, filesize=11.9 K 2024-11-17T07:15:36,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~152.20 KB/155856, currentSize=53.67 KB/54960 for cab1e460ee74e5eada90871d0516a7f5 in 89ms, sequenceid=193, compaction requested=true 2024-11-17T07:15:36,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:36,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:36,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:36,921 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:36,921 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:36,921 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:36,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:36,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:36,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:36,922 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:36,922 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45716 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:36,923 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/A is initiating minor compaction (all files) 2024-11-17T07:15:36,923 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/B is initiating minor compaction (all files) 2024-11-17T07:15:36,923 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/A in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:36,923 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/B in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:36,923 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/db60da51e5db4d7a9a8f0471e6f95c18, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/a1e49d8285794e219fa5bfc8da1c470e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/d43dc67744fa40ab942d8496016fc8fd] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=35.4 K 2024-11-17T07:15:36,923 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f41c4c052c474c59be0197859376f5a9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/fcb2d11fbd064ab4af599923dc3e7d48, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ec6e6e2c112545c89f862a168b1daf32] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=44.6 K 2024-11-17T07:15:36,923 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting f41c4c052c474c59be0197859376f5a9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731827734745 2024-11-17T07:15:36,923 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting db60da51e5db4d7a9a8f0471e6f95c18, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731827734745 2024-11-17T07:15:36,923 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcb2d11fbd064ab4af599923dc3e7d48, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731827735145 2024-11-17T07:15:36,923 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a1e49d8285794e219fa5bfc8da1c470e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731827735145 2024-11-17T07:15:36,924 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec6e6e2c112545c89f862a168b1daf32, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731827736813 2024-11-17T07:15:36,924 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting d43dc67744fa40ab942d8496016fc8fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731827736813 2024-11-17T07:15:36,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:36,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=60.38 KB heapSize=161.16 KB 2024-11-17T07:15:36,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:36,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:36,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:36,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:36,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:36,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:36,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T07:15:36,935 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:36,936 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/11b728cda6a542569c2bc62f5b340aad is 50, key is test_row_0/A:col10/1731827736831/Put/seqid=0 2024-11-17T07:15:36,937 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#B#compaction#406 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:36,937 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/f3412a8020d54e4f8da23492446fdee4 is 50, key is test_row_0/B:col10/1731827736831/Put/seqid=0 2024-11-17T07:15:36,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/f73de5fb94fb4539896d8aa151e8b3b0 is 50, key is test_row_0/A:col10/1731827736924/Put/seqid=0 2024-11-17T07:15:36,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742305_1481 (size=12357) 2024-11-17T07:15:36,959 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/11b728cda6a542569c2bc62f5b340aad as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/11b728cda6a542569c2bc62f5b340aad 2024-11-17T07:15:36,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742307_1483 (size=14541) 2024-11-17T07:15:36,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/f73de5fb94fb4539896d8aa151e8b3b0 2024-11-17T07:15:36,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742306_1482 (size=12357) 2024-11-17T07:15:36,971 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/A of cab1e460ee74e5eada90871d0516a7f5 into 11b728cda6a542569c2bc62f5b340aad(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:36,971 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:36,971 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/A, priority=13, startTime=1731827736921; duration=0sec 2024-11-17T07:15:36,971 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:36,971 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:A 2024-11-17T07:15:36,971 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:36,973 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36256 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:36,973 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/C is initiating minor compaction (all files) 2024-11-17T07:15:36,973 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/C in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:36,973 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/331deeae69a244d48fd93c38f3903165, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/eda253111c814b0e96189a4926063b7d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/563030a4d83844709472fdf56f64f53d] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=35.4 K 2024-11-17T07:15:36,973 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 331deeae69a244d48fd93c38f3903165, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731827734745 2024-11-17T07:15:36,974 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/4af227e0c6734eb58ac2ef5a72c153cf is 50, key is test_row_0/B:col10/1731827736924/Put/seqid=0 2024-11-17T07:15:36,975 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting eda253111c814b0e96189a4926063b7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731827735145 2024-11-17T07:15:36,975 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/f3412a8020d54e4f8da23492446fdee4 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/f3412a8020d54e4f8da23492446fdee4 2024-11-17T07:15:36,975 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 563030a4d83844709472fdf56f64f53d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731827736813 2024-11-17T07:15:36,979 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/B of cab1e460ee74e5eada90871d0516a7f5 into f3412a8020d54e4f8da23492446fdee4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:36,979 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:36,979 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/B, priority=13, startTime=1731827736921; duration=0sec 2024-11-17T07:15:36,979 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:36,979 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:B 2024-11-17T07:15:36,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742308_1484 (size=12151) 2024-11-17T07:15:36,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/4af227e0c6734eb58ac2ef5a72c153cf 2024-11-17T07:15:36,987 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#C#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:36,987 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/28514b6b753c4bc7ba058fbed59e7b24 is 50, key is test_row_0/C:col10/1731827736831/Put/seqid=0 2024-11-17T07:15:37,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742309_1485 (size=12357) 2024-11-17T07:15:37,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/eca19a1a163e41c48223d6558856c839 is 50, key is test_row_0/C:col10/1731827736924/Put/seqid=0 2024-11-17T07:15:37,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742310_1486 (size=12151) 2024-11-17T07:15:37,011 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/28514b6b753c4bc7ba058fbed59e7b24 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/28514b6b753c4bc7ba058fbed59e7b24 2024-11-17T07:15:37,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/eca19a1a163e41c48223d6558856c839 2024-11-17T07:15:37,015 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/C of cab1e460ee74e5eada90871d0516a7f5 into 28514b6b753c4bc7ba058fbed59e7b24(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:37,015 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:37,015 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/C, priority=13, startTime=1731827736922; duration=0sec 2024-11-17T07:15:37,016 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:37,016 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:C 2024-11-17T07:15:37,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/f73de5fb94fb4539896d8aa151e8b3b0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f73de5fb94fb4539896d8aa151e8b3b0 2024-11-17T07:15:37,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f73de5fb94fb4539896d8aa151e8b3b0, entries=200, sequenceid=218, filesize=14.2 K 2024-11-17T07:15:37,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/4af227e0c6734eb58ac2ef5a72c153cf as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/4af227e0c6734eb58ac2ef5a72c153cf 2024-11-17T07:15:37,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/4af227e0c6734eb58ac2ef5a72c153cf, entries=150, sequenceid=218, filesize=11.9 K 2024-11-17T07:15:37,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827797023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/eca19a1a163e41c48223d6558856c839 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/eca19a1a163e41c48223d6558856c839 2024-11-17T07:15:37,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827797024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827737003, max timestamp=1731827737026Num uniques 859; 2024-11-17T07:15:37,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:15:37,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827737003, max timestamp=1731827737026Num uniques 859; 2024-11-17T07:15:37,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827737003, max timestamp=1731827737026Num uniques 859; 2024-11-17T07:15:37,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:15:37,030 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:15:37,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/eca19a1a163e41c48223d6558856c839, entries=150, sequenceid=218, filesize=11.9 K 2024-11-17T07:15:37,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~161.11 KB/164976, currentSize=40.25 KB/41220 for cab1e460ee74e5eada90871d0516a7f5 in 108ms, sequenceid=218, compaction requested=false 2024-11-17T07:15:37,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:37,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:37,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=53.67 KB heapSize=136.55 KB 2024-11-17T07:15:37,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:37,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:37,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:37,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:37,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:37,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:37,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/0322287dcd404beba8f907b0378d9ee6 is 50, key is test_row_0/A:col10/1731827737037/Put/seqid=0 2024-11-17T07:15:37,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742311_1487 (size=14541) 2024-11-17T07:15:37,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/0322287dcd404beba8f907b0378d9ee6 2024-11-17T07:15:37,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/218256ade37048edbdeb95dd3807c614 is 50, key is test_row_0/B:col10/1731827737037/Put/seqid=0 2024-11-17T07:15:37,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742312_1488 (size=12151) 2024-11-17T07:15:37,067 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/218256ade37048edbdeb95dd3807c614 2024-11-17T07:15:37,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T07:15:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:37,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/41344710655944fca645ad84038c1854 is 50, key is test_row_0/C:col10/1731827737037/Put/seqid=0 2024-11-17T07:15:37,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742313_1489 (size=12151) 2024-11-17T07:15:37,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827737123, max timestamp=1731827737139Num uniques 859; 2024-11-17T07:15:37,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827737123, max timestamp=1731827737139Num uniques 859; 2024-11-17T07:15:37,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731827737123, max timestamp=1731827737139Num uniques 859; 2024-11-17T07:15:37,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:15:37,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:15:37,143 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-17T07:15:37,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827797167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827797174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827797178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827797180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827797181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T07:15:37,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:37,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,225 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T07:15:37,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827797287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827797288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,299 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827797297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827797297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,307 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827797307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T07:15:37,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:37,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/41344710655944fca645ad84038c1854 2024-11-17T07:15:37,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/0322287dcd404beba8f907b0378d9ee6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/0322287dcd404beba8f907b0378d9ee6 2024-11-17T07:15:37,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/0322287dcd404beba8f907b0378d9ee6, entries=200, sequenceid=245, filesize=14.2 K 2024-11-17T07:15:37,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/218256ade37048edbdeb95dd3807c614 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/218256ade37048edbdeb95dd3807c614 2024-11-17T07:15:37,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/218256ade37048edbdeb95dd3807c614, entries=150, sequenceid=245, filesize=11.9 K 2024-11-17T07:15:37,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/41344710655944fca645ad84038c1854 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/41344710655944fca645ad84038c1854 2024-11-17T07:15:37,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/41344710655944fca645ad84038c1854, entries=150, sequenceid=245, filesize=11.9 K 2024-11-17T07:15:37,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~156.42 KB/160176, currentSize=127.47 KB/130530 for cab1e460ee74e5eada90871d0516a7f5 in 461ms, sequenceid=245, compaction requested=true 2024-11-17T07:15:37,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:37,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:37,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:37,499 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:37,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:37,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:37,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:37,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:37,499 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:37,500 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41439 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:37,500 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:37,500 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/A is initiating minor compaction (all files) 2024-11-17T07:15:37,500 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/B is initiating minor compaction (all files) 2024-11-17T07:15:37,500 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/A in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,500 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/B in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,500 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/11b728cda6a542569c2bc62f5b340aad, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f73de5fb94fb4539896d8aa151e8b3b0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/0322287dcd404beba8f907b0378d9ee6] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=40.5 K 2024-11-17T07:15:37,501 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/f3412a8020d54e4f8da23492446fdee4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/4af227e0c6734eb58ac2ef5a72c153cf, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/218256ade37048edbdeb95dd3807c614] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=35.8 K 2024-11-17T07:15:37,501 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11b728cda6a542569c2bc62f5b340aad, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731827736813 2024-11-17T07:15:37,501 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting f3412a8020d54e4f8da23492446fdee4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731827736813 2024-11-17T07:15:37,501 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting f73de5fb94fb4539896d8aa151e8b3b0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731827736894 2024-11-17T07:15:37,501 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 4af227e0c6734eb58ac2ef5a72c153cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731827736914 2024-11-17T07:15:37,501 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0322287dcd404beba8f907b0378d9ee6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731827737020 2024-11-17T07:15:37,501 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 218256ade37048edbdeb95dd3807c614, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731827737020 2024-11-17T07:15:37,510 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#B#compaction#415 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:37,511 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/b895883b2c4340a294493047dd5624f1 is 50, key is test_row_0/B:col10/1731827737037/Put/seqid=0 2024-11-17T07:15:37,511 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#A#compaction#414 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:37,512 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/ced74b60a7b34405ab78fabdfd894c4f is 50, key is test_row_0/A:col10/1731827737037/Put/seqid=0 2024-11-17T07:15:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:37,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-17T07:15:37,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742314_1490 (size=12459) 2024-11-17T07:15:37,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:37,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:37,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:37,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:37,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:37,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:37,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742315_1491 (size=12459) 2024-11-17T07:15:37,528 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/b895883b2c4340a294493047dd5624f1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/b895883b2c4340a294493047dd5624f1 2024-11-17T07:15:37,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T07:15:37,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:37,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827797529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827797536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,541 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/ced74b60a7b34405ab78fabdfd894c4f as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ced74b60a7b34405ab78fabdfd894c4f 2024-11-17T07:15:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,545 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/A of cab1e460ee74e5eada90871d0516a7f5 into ced74b60a7b34405ab78fabdfd894c4f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:37,545 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:37,545 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/A, priority=13, startTime=1731827737499; duration=0sec 2024-11-17T07:15:37,545 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:37,545 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:A 2024-11-17T07:15:37,545 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:37,546 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/B of cab1e460ee74e5eada90871d0516a7f5 into b895883b2c4340a294493047dd5624f1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:37,546 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:37,546 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:37,546 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/B, priority=13, startTime=1731827737499; duration=0sec 2024-11-17T07:15:37,546 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/C is initiating minor compaction (all files) 2024-11-17T07:15:37,546 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/C in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,546 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:37,546 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:B 2024-11-17T07:15:37,546 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/28514b6b753c4bc7ba058fbed59e7b24, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/eca19a1a163e41c48223d6558856c839, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/41344710655944fca645ad84038c1854] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=35.8 K 2024-11-17T07:15:37,546 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28514b6b753c4bc7ba058fbed59e7b24, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731827736813 2024-11-17T07:15:37,547 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting eca19a1a163e41c48223d6558856c839, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1731827736914 2024-11-17T07:15:37,547 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41344710655944fca645ad84038c1854, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731827737020 2024-11-17T07:15:37,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/6f77304f1ccc49299b7a60895f59539e is 50, key is test_row_0/A:col10/1731827737519/Put/seqid=0 2024-11-17T07:15:37,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827797537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827797539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827797540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,562 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#C#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:37,562 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/d3462b79184e411496c78a6a1d0d46a7 is 50, key is test_row_0/C:col10/1731827737037/Put/seqid=0 2024-11-17T07:15:37,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742316_1492 (size=17181) 2024-11-17T07:15:37,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/6f77304f1ccc49299b7a60895f59539e 2024-11-17T07:15:37,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/89826a8566e9416db24e279a47aa4d17 is 50, key is test_row_0/B:col10/1731827737519/Put/seqid=0 2024-11-17T07:15:37,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742317_1493 (size=12459) 2024-11-17T07:15:37,633 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/d3462b79184e411496c78a6a1d0d46a7 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/d3462b79184e411496c78a6a1d0d46a7 2024-11-17T07:15:37,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742318_1494 (size=12301) 2024-11-17T07:15:37,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/89826a8566e9416db24e279a47aa4d17 2024-11-17T07:15:37,640 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/C of cab1e460ee74e5eada90871d0516a7f5 into d3462b79184e411496c78a6a1d0d46a7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:37,641 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:37,641 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/C, priority=13, startTime=1731827737499; duration=0sec 2024-11-17T07:15:37,641 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:37,641 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:C 2024-11-17T07:15:37,643 WARN [regionserver/1dd62e24b303:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-17T07:15:37,644 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f45cd89b0011470d97df1fddc5c2a944, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6d3bb7d6e48743139e5f37fefeb19477, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/30cfd9c3ed7c40baa74aa4e4477387f4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f41c4c052c474c59be0197859376f5a9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/fcb2d11fbd064ab4af599923dc3e7d48, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ec6e6e2c112545c89f862a168b1daf32, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/11b728cda6a542569c2bc62f5b340aad, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f73de5fb94fb4539896d8aa151e8b3b0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/0322287dcd404beba8f907b0378d9ee6] to archive 2024-11-17T07:15:37,648 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:15:37,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/4091a5365f784606b243b3e6b7116c21 is 50, key is test_row_0/C:col10/1731827737519/Put/seqid=0 2024-11-17T07:15:37,663 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f45cd89b0011470d97df1fddc5c2a944 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f45cd89b0011470d97df1fddc5c2a944 2024-11-17T07:15:37,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827797647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742319_1495 (size=12301) 2024-11-17T07:15:37,671 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6d3bb7d6e48743139e5f37fefeb19477 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6d3bb7d6e48743139e5f37fefeb19477 2024-11-17T07:15:37,672 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/30cfd9c3ed7c40baa74aa4e4477387f4 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/30cfd9c3ed7c40baa74aa4e4477387f4 2024-11-17T07:15:37,673 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f41c4c052c474c59be0197859376f5a9 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f41c4c052c474c59be0197859376f5a9 2024-11-17T07:15:37,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827797667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827797667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,674 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/fcb2d11fbd064ab4af599923dc3e7d48 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/fcb2d11fbd064ab4af599923dc3e7d48 2024-11-17T07:15:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827797667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,675 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,675 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ec6e6e2c112545c89f862a168b1daf32 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ec6e6e2c112545c89f862a168b1daf32 2024-11-17T07:15:37,676 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/11b728cda6a542569c2bc62f5b340aad to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/11b728cda6a542569c2bc62f5b340aad 2024-11-17T07:15:37,677 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f73de5fb94fb4539896d8aa151e8b3b0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f73de5fb94fb4539896d8aa151e8b3b0 2024-11-17T07:15:37,679 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/0322287dcd404beba8f907b0378d9ee6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/0322287dcd404beba8f907b0378d9ee6 2024-11-17T07:15:37,681 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/8854e5a14dd54560ad33e982bfdca402, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c1c1289beb634ba3af2a318a447f4e2d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/db60da51e5db4d7a9a8f0471e6f95c18, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/d1198d2d6c84429cbb22006e31fc584e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/a1e49d8285794e219fa5bfc8da1c470e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/f3412a8020d54e4f8da23492446fdee4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/d43dc67744fa40ab942d8496016fc8fd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/4af227e0c6734eb58ac2ef5a72c153cf, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/218256ade37048edbdeb95dd3807c614] to archive 2024-11-17T07:15:37,682 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:15:37,683 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/8854e5a14dd54560ad33e982bfdca402 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/8854e5a14dd54560ad33e982bfdca402 2024-11-17T07:15:37,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T07:15:37,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,684 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c1c1289beb634ba3af2a318a447f4e2d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c1c1289beb634ba3af2a318a447f4e2d 2024-11-17T07:15:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,686 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/db60da51e5db4d7a9a8f0471e6f95c18 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/db60da51e5db4d7a9a8f0471e6f95c18 2024-11-17T07:15:37,687 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/d1198d2d6c84429cbb22006e31fc584e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/d1198d2d6c84429cbb22006e31fc584e 2024-11-17T07:15:37,687 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/a1e49d8285794e219fa5bfc8da1c470e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/a1e49d8285794e219fa5bfc8da1c470e 2024-11-17T07:15:37,688 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/f3412a8020d54e4f8da23492446fdee4 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/f3412a8020d54e4f8da23492446fdee4 2024-11-17T07:15:37,689 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/d43dc67744fa40ab942d8496016fc8fd to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/d43dc67744fa40ab942d8496016fc8fd 2024-11-17T07:15:37,690 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/4af227e0c6734eb58ac2ef5a72c153cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/4af227e0c6734eb58ac2ef5a72c153cf 2024-11-17T07:15:37,691 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/218256ade37048edbdeb95dd3807c614 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/218256ade37048edbdeb95dd3807c614 2024-11-17T07:15:37,693 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/09fb35ad8d2c44da8999a41fff409383, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/0a45d419bbcd4a94bf8f32a8285fa118, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/331deeae69a244d48fd93c38f3903165, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/32793abb413848ee87e3348b909aaf3e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/eda253111c814b0e96189a4926063b7d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/28514b6b753c4bc7ba058fbed59e7b24, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/563030a4d83844709472fdf56f64f53d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/eca19a1a163e41c48223d6558856c839, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/41344710655944fca645ad84038c1854] to archive 2024-11-17T07:15:37,694 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:15:37,695 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/09fb35ad8d2c44da8999a41fff409383 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/09fb35ad8d2c44da8999a41fff409383 2024-11-17T07:15:37,696 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/0a45d419bbcd4a94bf8f32a8285fa118 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/0a45d419bbcd4a94bf8f32a8285fa118 2024-11-17T07:15:37,697 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/331deeae69a244d48fd93c38f3903165 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/331deeae69a244d48fd93c38f3903165 2024-11-17T07:15:37,697 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/32793abb413848ee87e3348b909aaf3e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/32793abb413848ee87e3348b909aaf3e 2024-11-17T07:15:37,698 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/eda253111c814b0e96189a4926063b7d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/eda253111c814b0e96189a4926063b7d 2024-11-17T07:15:37,699 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/28514b6b753c4bc7ba058fbed59e7b24 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/28514b6b753c4bc7ba058fbed59e7b24 2024-11-17T07:15:37,700 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/563030a4d83844709472fdf56f64f53d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/563030a4d83844709472fdf56f64f53d 2024-11-17T07:15:37,701 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/eca19a1a163e41c48223d6558856c839 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/eca19a1a163e41c48223d6558856c839 2024-11-17T07:15:37,702 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/1dd62e24b303:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/41344710655944fca645ad84038c1854 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/41344710655944fca645ad84038c1854 2024-11-17T07:15:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T07:15:37,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T07:15:37,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:37,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827797847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,853 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827797877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827797877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827797877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827797877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:37,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:37,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T07:15:37,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:37,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:37,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:38,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/4091a5365f784606b243b3e6b7116c21 2024-11-17T07:15:38,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/6f77304f1ccc49299b7a60895f59539e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6f77304f1ccc49299b7a60895f59539e 2024-11-17T07:15:38,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6f77304f1ccc49299b7a60895f59539e, entries=250, sequenceid=281, filesize=16.8 K 2024-11-17T07:15:38,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/89826a8566e9416db24e279a47aa4d17 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/89826a8566e9416db24e279a47aa4d17 2024-11-17T07:15:38,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/89826a8566e9416db24e279a47aa4d17, entries=150, sequenceid=281, filesize=12.0 K 2024-11-17T07:15:38,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/4091a5365f784606b243b3e6b7116c21 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/4091a5365f784606b243b3e6b7116c21 2024-11-17T07:15:38,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/4091a5365f784606b243b3e6b7116c21, entries=150, sequenceid=281, filesize=12.0 K 2024-11-17T07:15:38,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for cab1e460ee74e5eada90871d0516a7f5 in 563ms, sequenceid=281, compaction requested=false 2024-11-17T07:15:38,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:38,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-17T07:15:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:38,142 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-17T07:15:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:38,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/7b5c76135d8c4a6aacdd0f4976306a3f is 50, key is test_row_0/A:col10/1731827737535/Put/seqid=0 2024-11-17T07:15:38,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742320_1496 (size=12301) 2024-11-17T07:15:38,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:38,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:38,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731827738217, max timestamp=1731827738234Num uniques 859; 2024-11-17T07:15:38,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T07:15:38,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731827738217, max timestamp=1731827738234Num uniques 859; 2024-11-17T07:15:38,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731827738217, max timestamp=1731827738234Num uniques 859; 2024-11-17T07:15:38,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T07:15:38,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-17T07:15:38,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827798277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,285 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827798278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827798279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827798281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,297 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827798367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,377 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827798387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827798397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827798407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827798407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,551 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/7b5c76135d8c4a6aacdd0f4976306a3f 2024-11-17T07:15:38,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/c8b151778dc34c77939b35446ee51230 is 50, key is test_row_0/B:col10/1731827737535/Put/seqid=0 2024-11-17T07:15:38,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742321_1497 (size=12301) 2024-11-17T07:15:38,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827798607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827798617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827798627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827798627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T07:15:38,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827798927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827798937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827798937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827798937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:38,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:38,961 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/c8b151778dc34c77939b35446ee51230 2024-11-17T07:15:38,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/e3c96c88cf14468e938a702868e92d95 is 50, key is test_row_0/C:col10/1731827737535/Put/seqid=0 2024-11-17T07:15:38,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742322_1498 (size=12301) 2024-11-17T07:15:39,371 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/e3c96c88cf14468e938a702868e92d95 2024-11-17T07:15:39,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/7b5c76135d8c4a6aacdd0f4976306a3f as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/7b5c76135d8c4a6aacdd0f4976306a3f 2024-11-17T07:15:39,378 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/7b5c76135d8c4a6aacdd0f4976306a3f, entries=150, sequenceid=293, filesize=12.0 K 2024-11-17T07:15:39,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/c8b151778dc34c77939b35446ee51230 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c8b151778dc34c77939b35446ee51230 2024-11-17T07:15:39,383 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c8b151778dc34c77939b35446ee51230, entries=150, sequenceid=293, filesize=12.0 K 2024-11-17T07:15:39,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/e3c96c88cf14468e938a702868e92d95 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/e3c96c88cf14468e938a702868e92d95 2024-11-17T07:15:39,386 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/e3c96c88cf14468e938a702868e92d95, entries=150, sequenceid=293, filesize=12.0 K 2024-11-17T07:15:39,387 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for cab1e460ee74e5eada90871d0516a7f5 in 1245ms, sequenceid=293, compaction requested=true 2024-11-17T07:15:39,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:39,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:39,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-17T07:15:39,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-17T07:15:39,389 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-17T07:15:39,389 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7780 sec 2024-11-17T07:15:39,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.7830 sec 2024-11-17T07:15:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:39,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-17T07:15:39,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:39,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:39,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:39,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:39,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:39,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:39,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/f6234cb3f1a341849d45c4cec64e6288 is 50, key is test_row_0/A:col10/1731827738276/Put/seqid=0 2024-11-17T07:15:39,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742323_1499 (size=14741) 2024-11-17T07:15:39,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:39,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827799442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:39,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827799447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:39,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827799447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827799447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:39,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827799448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:39,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:39,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827799556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:39,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:39,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827799777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:39,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/f6234cb3f1a341849d45c4cec64e6288 2024-11-17T07:15:39,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/f2deb5ddce5640bab1dd7c506ca3f480 is 50, key is test_row_0/B:col10/1731827738276/Put/seqid=0 2024-11-17T07:15:39,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742324_1500 (size=12301) 2024-11-17T07:15:40,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827800097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:40,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:40,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/f2deb5ddce5640bab1dd7c506ca3f480 2024-11-17T07:15:40,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/8d9793e75ad5404ca7c5592f5048da88 is 50, key is test_row_0/C:col10/1731827738276/Put/seqid=0 2024-11-17T07:15:40,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742325_1501 (size=12301) 2024-11-17T07:15:40,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:40,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827800457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:40,461 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:40,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:40,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827800467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:40,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827800467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:40,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:40,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827800468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:40,477 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:40,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:40,477 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:40,477 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:40,478 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:40,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827800617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:40,624 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:40,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/8d9793e75ad5404ca7c5592f5048da88 2024-11-17T07:15:40,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/f6234cb3f1a341849d45c4cec64e6288 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f6234cb3f1a341849d45c4cec64e6288 2024-11-17T07:15:40,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f6234cb3f1a341849d45c4cec64e6288, entries=200, sequenceid=330, filesize=14.4 K 2024-11-17T07:15:40,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/f2deb5ddce5640bab1dd7c506ca3f480 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/f2deb5ddce5640bab1dd7c506ca3f480 2024-11-17T07:15:40,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/f2deb5ddce5640bab1dd7c506ca3f480, entries=150, sequenceid=330, filesize=12.0 K 2024-11-17T07:15:40,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/8d9793e75ad5404ca7c5592f5048da88 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/8d9793e75ad5404ca7c5592f5048da88 2024-11-17T07:15:40,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/8d9793e75ad5404ca7c5592f5048da88, entries=150, sequenceid=330, filesize=12.0 K 2024-11-17T07:15:40,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for cab1e460ee74e5eada90871d0516a7f5 in 1253ms, sequenceid=330, compaction requested=true 2024-11-17T07:15:40,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:40,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:40,648 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:40,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:40,649 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:15:40,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:40,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:40,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:40,649 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:15:40,650 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:15:40,650 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:15:40,650 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/B is initiating minor compaction (all files) 2024-11-17T07:15:40,650 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/A is initiating minor compaction (all files) 2024-11-17T07:15:40,650 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/B in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:40,650 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/A in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:40,650 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/b895883b2c4340a294493047dd5624f1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/89826a8566e9416db24e279a47aa4d17, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c8b151778dc34c77939b35446ee51230, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/f2deb5ddce5640bab1dd7c506ca3f480] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=48.2 K 2024-11-17T07:15:40,650 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ced74b60a7b34405ab78fabdfd894c4f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6f77304f1ccc49299b7a60895f59539e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/7b5c76135d8c4a6aacdd0f4976306a3f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f6234cb3f1a341849d45c4cec64e6288] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=55.4 K 2024-11-17T07:15:40,651 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting b895883b2c4340a294493047dd5624f1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731827737020 2024-11-17T07:15:40,651 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting ced74b60a7b34405ab78fabdfd894c4f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731827737020 2024-11-17T07:15:40,651 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f77304f1ccc49299b7a60895f59539e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731827737177 2024-11-17T07:15:40,651 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 89826a8566e9416db24e279a47aa4d17, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731827737177 2024-11-17T07:15:40,651 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b5c76135d8c4a6aacdd0f4976306a3f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731827737535 2024-11-17T07:15:40,651 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting c8b151778dc34c77939b35446ee51230, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731827737535 2024-11-17T07:15:40,652 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting f2deb5ddce5640bab1dd7c506ca3f480, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731827738276 2024-11-17T07:15:40,652 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6234cb3f1a341849d45c4cec64e6288, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731827738262 2024-11-17T07:15:40,660 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#A#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:40,661 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/35ff17ba7a5640ffb377ba8f55cd5694 is 50, key is test_row_0/A:col10/1731827738276/Put/seqid=0 2024-11-17T07:15:40,661 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#B#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:40,661 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/6dffd28afba04b1fa32eaa0e5c987d26 is 50, key is test_row_0/B:col10/1731827738276/Put/seqid=0 2024-11-17T07:15:40,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742326_1502 (size=12439) 2024-11-17T07:15:40,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742327_1503 (size=12439) 2024-11-17T07:15:40,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-17T07:15:40,767 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:40,768 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:40,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-17T07:15:40,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T07:15:40,769 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:40,770 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:40,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:40,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T07:15:40,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-17T07:15:40,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:40,923 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-17T07:15:40,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:40,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:40,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:40,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:40,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:40,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:40,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/a07eeba9130a49d5bb7c4f34b867249a is 50, key is test_row_0/A:col10/1731827739441/Put/seqid=0 2024-11-17T07:15:40,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742328_1504 (size=12301) 2024-11-17T07:15:41,075 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/35ff17ba7a5640ffb377ba8f55cd5694 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/35ff17ba7a5640ffb377ba8f55cd5694 2024-11-17T07:15:41,076 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/6dffd28afba04b1fa32eaa0e5c987d26 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/6dffd28afba04b1fa32eaa0e5c987d26 2024-11-17T07:15:41,080 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/A of cab1e460ee74e5eada90871d0516a7f5 into 35ff17ba7a5640ffb377ba8f55cd5694(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:41,080 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:41,080 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/A, priority=12, startTime=1731827740648; duration=0sec 2024-11-17T07:15:41,080 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:41,080 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:A 2024-11-17T07:15:41,080 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:15:41,081 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/B of cab1e460ee74e5eada90871d0516a7f5 into 6dffd28afba04b1fa32eaa0e5c987d26(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:41,081 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:41,081 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/B, priority=12, startTime=1731827740648; duration=0sec 2024-11-17T07:15:41,081 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:41,081 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:B 2024-11-17T07:15:41,081 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49362 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:15:41,081 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/C is initiating minor compaction (all files) 2024-11-17T07:15:41,081 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/C in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:41,082 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/d3462b79184e411496c78a6a1d0d46a7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/4091a5365f784606b243b3e6b7116c21, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/e3c96c88cf14468e938a702868e92d95, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/8d9793e75ad5404ca7c5592f5048da88] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=48.2 K 2024-11-17T07:15:41,082 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3462b79184e411496c78a6a1d0d46a7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731827737020 2024-11-17T07:15:41,082 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4091a5365f784606b243b3e6b7116c21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731827737177 2024-11-17T07:15:41,083 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3c96c88cf14468e938a702868e92d95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731827737535 2024-11-17T07:15:41,083 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d9793e75ad5404ca7c5592f5048da88, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731827738276 2024-11-17T07:15:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T07:15:41,095 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#C#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:41,095 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/1c0c51dbd3434e21aa60aef12a2857c3 is 50, key is test_row_0/C:col10/1731827738276/Put/seqid=0 2024-11-17T07:15:41,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742329_1505 (size=12439) 2024-11-17T07:15:41,107 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/1c0c51dbd3434e21aa60aef12a2857c3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/1c0c51dbd3434e21aa60aef12a2857c3 2024-11-17T07:15:41,112 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/C of cab1e460ee74e5eada90871d0516a7f5 into 1c0c51dbd3434e21aa60aef12a2857c3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:41,112 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:41,112 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/C, priority=12, startTime=1731827740649; duration=0sec 2024-11-17T07:15:41,112 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:41,112 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:C 2024-11-17T07:15:41,333 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/a07eeba9130a49d5bb7c4f34b867249a 2024-11-17T07:15:41,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/9a4e60827a984641a1c0a0c49ebcb800 is 50, key is test_row_0/B:col10/1731827739441/Put/seqid=0 2024-11-17T07:15:41,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742330_1506 (size=12301) 2024-11-17T07:15:41,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T07:15:41,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:41,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:41,746 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/9a4e60827a984641a1c0a0c49ebcb800 2024-11-17T07:15:41,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/bfd3cd10474d408d8f50834dbc30619a is 50, key is test_row_0/C:col10/1731827739441/Put/seqid=0 2024-11-17T07:15:41,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827741677, max timestamp=1731827741753Num uniques 859; 2024-11-17T07:15:41,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:41,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827741677, max timestamp=1731827741753Num uniques 859; 2024-11-17T07:15:41,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827741677, max timestamp=1731827741753Num uniques 859; 2024-11-17T07:15:41,756 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:41,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:41,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742331_1507 (size=12301) 2024-11-17T07:15:41,765 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/bfd3cd10474d408d8f50834dbc30619a 2024-11-17T07:15:41,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/a07eeba9130a49d5bb7c4f34b867249a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/a07eeba9130a49d5bb7c4f34b867249a 2024-11-17T07:15:41,773 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/a07eeba9130a49d5bb7c4f34b867249a, entries=150, sequenceid=338, filesize=12.0 K 2024-11-17T07:15:41,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/9a4e60827a984641a1c0a0c49ebcb800 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/9a4e60827a984641a1c0a0c49ebcb800 2024-11-17T07:15:41,776 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/9a4e60827a984641a1c0a0c49ebcb800, entries=150, sequenceid=338, filesize=12.0 K 2024-11-17T07:15:41,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/bfd3cd10474d408d8f50834dbc30619a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/bfd3cd10474d408d8f50834dbc30619a 2024-11-17T07:15:41,780 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/bfd3cd10474d408d8f50834dbc30619a, entries=150, sequenceid=338, filesize=12.0 K 2024-11-17T07:15:41,780 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=46.96 KB/48090 for cab1e460ee74e5eada90871d0516a7f5 in 857ms, sequenceid=338, compaction requested=false 2024-11-17T07:15:41,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:41,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:41,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-17T07:15:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-17T07:15:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:41,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-17T07:15:41,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:41,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:41,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:41,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:41,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:41,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:41,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-17T07:15:41,784 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0110 sec 2024-11-17T07:15:41,785 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.0160 sec 2024-11-17T07:15:41,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/b9c3c9ae968946949e0e35987b757ef0 is 50, key is test_row_0/A:col10/1731827741777/Put/seqid=0 2024-11-17T07:15:41,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742332_1508 (size=17181) 2024-11-17T07:15:41,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/b9c3c9ae968946949e0e35987b757ef0 2024-11-17T07:15:41,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/6bc35d50f8c3462bbff799c4be8131e3 is 50, key is test_row_0/B:col10/1731827741777/Put/seqid=0 2024-11-17T07:15:41,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742333_1509 (size=12301) 2024-11-17T07:15:41,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-17T07:15:41,908 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:41,908 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-17T07:15:41,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T07:15:41,909 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:41,910 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:41,910 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:41,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827741881, max timestamp=1731827741918Num uniques 859; 2024-11-17T07:15:41,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:41,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827741881, max timestamp=1731827741918Num uniques 859; 2024-11-17T07:15:41,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827741881, max timestamp=1731827741918Num uniques 859; 2024-11-17T07:15:41,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:41,921 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T07:15:42,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T07:15:42,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:42,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:42,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:42,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:42,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:42,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:42,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:42,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827802081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:42,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,083 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:42,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827802187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:42,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/6bc35d50f8c3462bbff799c4be8131e3 2024-11-17T07:15:42,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/2d8d8832d56549bfbbd9c06b5736bd3a is 50, key is test_row_0/C:col10/1731827741777/Put/seqid=0 2024-11-17T07:15:42,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742334_1510 (size=12301) 2024-11-17T07:15:42,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T07:15:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:42,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:42,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T07:15:42,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T07:15:42,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:42,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:42,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:42,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:42,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:42,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:42,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:42,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827802407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:42,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:42,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827802477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:42,487 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,488 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:42,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827802487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:42,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:42,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827802487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:42,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:42,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827802487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:42,495 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,496 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,496 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:42,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:42,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T07:15:42,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:42,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:42,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:42,521 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:42,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:42,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:42,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T07:15:42,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/2d8d8832d56549bfbbd9c06b5736bd3a 2024-11-17T07:15:42,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/b9c3c9ae968946949e0e35987b757ef0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/b9c3c9ae968946949e0e35987b757ef0 2024-11-17T07:15:42,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/b9c3c9ae968946949e0e35987b757ef0, entries=250, sequenceid=366, filesize=16.8 K 2024-11-17T07:15:42,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/6bc35d50f8c3462bbff799c4be8131e3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/6bc35d50f8c3462bbff799c4be8131e3 2024-11-17T07:15:42,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/6bc35d50f8c3462bbff799c4be8131e3, entries=150, sequenceid=366, filesize=12.0 K 2024-11-17T07:15:42,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/2d8d8832d56549bfbbd9c06b5736bd3a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/2d8d8832d56549bfbbd9c06b5736bd3a 2024-11-17T07:15:42,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/2d8d8832d56549bfbbd9c06b5736bd3a, entries=150, sequenceid=366, filesize=12.0 K 2024-11-17T07:15:42,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for cab1e460ee74e5eada90871d0516a7f5 in 844ms, sequenceid=366, compaction requested=true 2024-11-17T07:15:42,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:42,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:42,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:42,626 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:42,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:42,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:42,626 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:42,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:42,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:42,627 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41921 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:42,627 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:42,627 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/A is initiating minor compaction (all files) 2024-11-17T07:15:42,627 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/B is initiating minor compaction (all files) 2024-11-17T07:15:42,627 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/A in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:42,627 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/B in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:42,627 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/35ff17ba7a5640ffb377ba8f55cd5694, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/a07eeba9130a49d5bb7c4f34b867249a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/b9c3c9ae968946949e0e35987b757ef0] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=40.9 K 2024-11-17T07:15:42,627 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/6dffd28afba04b1fa32eaa0e5c987d26, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/9a4e60827a984641a1c0a0c49ebcb800, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/6bc35d50f8c3462bbff799c4be8131e3] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=36.2 K 2024-11-17T07:15:42,628 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35ff17ba7a5640ffb377ba8f55cd5694, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731827738276 2024-11-17T07:15:42,628 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 6dffd28afba04b1fa32eaa0e5c987d26, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731827738276 2024-11-17T07:15:42,628 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a4e60827a984641a1c0a0c49ebcb800, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731827739410 2024-11-17T07:15:42,628 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting a07eeba9130a49d5bb7c4f34b867249a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731827739410 2024-11-17T07:15:42,628 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bc35d50f8c3462bbff799c4be8131e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731827741739 2024-11-17T07:15:42,628 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9c3c9ae968946949e0e35987b757ef0, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731827741739 2024-11-17T07:15:42,635 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#B#compaction#435 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:42,635 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/c83f32e8e9244486a98c94b858d3a0b9 is 50, key is test_row_0/B:col10/1731827741777/Put/seqid=0 2024-11-17T07:15:42,638 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#A#compaction#436 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:42,639 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/7460cfa5becd499aa9c08fec815662b5 is 50, key is test_row_0/A:col10/1731827741777/Put/seqid=0 2024-11-17T07:15:42,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742335_1511 (size=12541) 2024-11-17T07:15:42,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742336_1512 (size=12541) 2024-11-17T07:15:42,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-17T07:15:42,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:42,674 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-17T07:15:42,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:42,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:42,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:42,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:42,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/04ee62ee56c247668fb13204816af137 is 50, key is test_row_0/A:col10/1731827742080/Put/seqid=0 2024-11-17T07:15:42,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742337_1513 (size=12301) 2024-11-17T07:15:42,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:42,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:42,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827802782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:42,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:42,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827802897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:42,908 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:43,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T07:15:43,050 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/c83f32e8e9244486a98c94b858d3a0b9 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c83f32e8e9244486a98c94b858d3a0b9 2024-11-17T07:15:43,050 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/7460cfa5becd499aa9c08fec815662b5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/7460cfa5becd499aa9c08fec815662b5 2024-11-17T07:15:43,053 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/B of cab1e460ee74e5eada90871d0516a7f5 into c83f32e8e9244486a98c94b858d3a0b9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:43,054 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:43,054 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/B, priority=13, startTime=1731827742626; duration=0sec 2024-11-17T07:15:43,054 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:43,054 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:B 2024-11-17T07:15:43,054 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:43,054 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/A of cab1e460ee74e5eada90871d0516a7f5 into 7460cfa5becd499aa9c08fec815662b5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:43,054 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:43,054 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/A, priority=13, startTime=1731827742626; duration=0sec 2024-11-17T07:15:43,054 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:43,054 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:A 2024-11-17T07:15:43,054 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37041 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:43,055 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/C is initiating minor compaction (all files) 2024-11-17T07:15:43,055 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/C in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:43,055 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/1c0c51dbd3434e21aa60aef12a2857c3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/bfd3cd10474d408d8f50834dbc30619a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/2d8d8832d56549bfbbd9c06b5736bd3a] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=36.2 K 2024-11-17T07:15:43,055 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c0c51dbd3434e21aa60aef12a2857c3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731827738276 2024-11-17T07:15:43,055 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting bfd3cd10474d408d8f50834dbc30619a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731827739410 2024-11-17T07:15:43,055 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d8d8832d56549bfbbd9c06b5736bd3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731827741739 2024-11-17T07:15:43,063 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#C#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:43,064 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/24311c476f0143e4939a83eb012eac7d is 50, key is test_row_0/C:col10/1731827741777/Put/seqid=0 2024-11-17T07:15:43,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742338_1514 (size=12541) 2024-11-17T07:15:43,081 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/24311c476f0143e4939a83eb012eac7d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/24311c476f0143e4939a83eb012eac7d 2024-11-17T07:15:43,085 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/04ee62ee56c247668fb13204816af137 2024-11-17T07:15:43,090 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/C of cab1e460ee74e5eada90871d0516a7f5 into 24311c476f0143e4939a83eb012eac7d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:43,090 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:43,090 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/C, priority=13, startTime=1731827742626; duration=0sec 2024-11-17T07:15:43,090 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:43,090 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:C 2024-11-17T07:15:43,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/0f7512edf70e4771b1f46640a66d400b is 50, key is test_row_0/B:col10/1731827742080/Put/seqid=0 2024-11-17T07:15:43,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742339_1515 (size=12301) 2024-11-17T07:15:43,109 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/0f7512edf70e4771b1f46640a66d400b 2024-11-17T07:15:43,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:43,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827803122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:43,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:43,126 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:43,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/2a60c2d99f0e4a7a97b74c855844242b is 50, key is test_row_0/C:col10/1731827742080/Put/seqid=0 2024-11-17T07:15:43,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742340_1516 (size=12301) 2024-11-17T07:15:43,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:43,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827803437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:43,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:43,554 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/2a60c2d99f0e4a7a97b74c855844242b 2024-11-17T07:15:43,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/04ee62ee56c247668fb13204816af137 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/04ee62ee56c247668fb13204816af137 2024-11-17T07:15:43,561 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/04ee62ee56c247668fb13204816af137, entries=150, sequenceid=403, filesize=12.0 K 2024-11-17T07:15:43,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/0f7512edf70e4771b1f46640a66d400b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/0f7512edf70e4771b1f46640a66d400b 2024-11-17T07:15:43,570 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/0f7512edf70e4771b1f46640a66d400b, entries=150, sequenceid=403, filesize=12.0 K 2024-11-17T07:15:43,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/2a60c2d99f0e4a7a97b74c855844242b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/2a60c2d99f0e4a7a97b74c855844242b 2024-11-17T07:15:43,575 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/2a60c2d99f0e4a7a97b74c855844242b, entries=150, sequenceid=403, filesize=12.0 K 2024-11-17T07:15:43,575 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cab1e460ee74e5eada90871d0516a7f5 in 901ms, sequenceid=403, compaction requested=false 2024-11-17T07:15:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:43,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-17T07:15:43,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-17T07:15:43,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-17T07:15:43,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6670 sec 2024-11-17T07:15:43,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.6710 sec 2024-11-17T07:15:43,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:43,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:15:43,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:43,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:43,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:43,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:43,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:43,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:43,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/dccc937d9a8146d3bd8239676a0097f3 is 50, key is test_row_0/A:col10/1731827743957/Put/seqid=0 2024-11-17T07:15:43,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742341_1517 (size=12297) 2024-11-17T07:15:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-17T07:15:44,057 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:44,057 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-17T07:15:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T07:15:44,059 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:44,060 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:44,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:44,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827744031, max timestamp=1731827744071Num uniques 859; 2024-11-17T07:15:44,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827744031, max timestamp=1731827744071Num uniques 859; 2024-11-17T07:15:44,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:44,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:44,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827744031, max timestamp=1731827744071Num uniques 859; 2024-11-17T07:15:44,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T07:15:44,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:44,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827804188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:44,193 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:44,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T07:15:44,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:44,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:44,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:44,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:44,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:44,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827804297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:44,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T07:15:44,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:44,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:44,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:44,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:44,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:44,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/dccc937d9a8146d3bd8239676a0097f3 2024-11-17T07:15:44,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/bb3c6a1bea8648b2affca38df4129263 is 50, key is test_row_0/B:col10/1731827743957/Put/seqid=0 2024-11-17T07:15:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T07:15:44,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742342_1518 (size=9857) 2024-11-17T07:15:44,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/bb3c6a1bea8648b2affca38df4129263 2024-11-17T07:15:44,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/40e0100017d94b9082fd6b42cece6fa5 is 50, key is test_row_0/C:col10/1731827743957/Put/seqid=0 2024-11-17T07:15:44,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742343_1519 (size=9857) 2024-11-17T07:15:44,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T07:15:44,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:44,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:44,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:44,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:44,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:44,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:44,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:44,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827804507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:44,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:44,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:44,521 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:44,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T07:15:44,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:44,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:44,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:44,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:44,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:44,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:44,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T07:15:44,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/40e0100017d94b9082fd6b42cece6fa5 2024-11-17T07:15:44,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/dccc937d9a8146d3bd8239676a0097f3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/dccc937d9a8146d3bd8239676a0097f3 2024-11-17T07:15:44,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/dccc937d9a8146d3bd8239676a0097f3, entries=150, sequenceid=417, filesize=12.0 K 2024-11-17T07:15:44,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/bb3c6a1bea8648b2affca38df4129263 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/bb3c6a1bea8648b2affca38df4129263 2024-11-17T07:15:44,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/bb3c6a1bea8648b2affca38df4129263, entries=100, sequenceid=417, filesize=9.6 K 2024-11-17T07:15:44,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/40e0100017d94b9082fd6b42cece6fa5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/40e0100017d94b9082fd6b42cece6fa5 2024-11-17T07:15:44,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/40e0100017d94b9082fd6b42cece6fa5, entries=100, sequenceid=417, filesize=9.6 K 2024-11-17T07:15:44,805 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cab1e460ee74e5eada90871d0516a7f5 in 847ms, sequenceid=417, compaction requested=true 2024-11-17T07:15:44,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:44,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:44,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:44,805 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:44,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:44,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:44,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:44,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:44,805 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:44,806 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37139 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:44,806 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:44,806 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/A is initiating minor compaction (all files) 2024-11-17T07:15:44,806 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/B is initiating minor compaction (all files) 2024-11-17T07:15:44,806 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/A in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:44,806 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/B in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:44,806 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/7460cfa5becd499aa9c08fec815662b5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/04ee62ee56c247668fb13204816af137, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/dccc937d9a8146d3bd8239676a0097f3] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=36.3 K 2024-11-17T07:15:44,806 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c83f32e8e9244486a98c94b858d3a0b9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/0f7512edf70e4771b1f46640a66d400b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/bb3c6a1bea8648b2affca38df4129263] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=33.9 K 2024-11-17T07:15:44,806 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7460cfa5becd499aa9c08fec815662b5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731827741739 2024-11-17T07:15:44,806 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting c83f32e8e9244486a98c94b858d3a0b9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731827741739 2024-11-17T07:15:44,807 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04ee62ee56c247668fb13204816af137, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731827742047 2024-11-17T07:15:44,807 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f7512edf70e4771b1f46640a66d400b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731827742047 2024-11-17T07:15:44,807 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting dccc937d9a8146d3bd8239676a0097f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731827742760 2024-11-17T07:15:44,807 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting bb3c6a1bea8648b2affca38df4129263, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731827742781 2024-11-17T07:15:44,813 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:44,813 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/772e1a59944a4ee7b27cf75c16b11c70 is 50, key is test_row_0/B:col10/1731827743957/Put/seqid=0 2024-11-17T07:15:44,814 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#A#compaction#445 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:44,814 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/5d6f5d5c6eeb44db98fd6447b812fabf is 50, key is test_row_0/A:col10/1731827743957/Put/seqid=0 2024-11-17T07:15:44,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-17T07:15:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:44,824 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-17T07:15:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:44,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:44,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:44,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:44,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742344_1520 (size=12643) 2024-11-17T07:15:44,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742345_1521 (size=12643) 2024-11-17T07:15:44,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/c7fb406aef8641d0881a6e945c7847a1 is 50, key is test_row_0/A:col10/1731827744165/Put/seqid=0 2024-11-17T07:15:44,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742346_1522 (size=12301) 2024-11-17T07:15:44,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:44,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827804917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:44,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:45,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:45,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827805027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:45,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:45,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:45,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T07:15:45,240 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/772e1a59944a4ee7b27cf75c16b11c70 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/772e1a59944a4ee7b27cf75c16b11c70 2024-11-17T07:15:45,243 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/B of cab1e460ee74e5eada90871d0516a7f5 into 772e1a59944a4ee7b27cf75c16b11c70(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:45,243 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:45,244 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/B, priority=13, startTime=1731827744805; duration=0sec 2024-11-17T07:15:45,244 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:45,244 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:B 2024-11-17T07:15:45,244 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:45,244 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34699 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:45,245 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/C is initiating minor compaction (all files) 2024-11-17T07:15:45,245 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/C in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:45,245 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/24311c476f0143e4939a83eb012eac7d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/2a60c2d99f0e4a7a97b74c855844242b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/40e0100017d94b9082fd6b42cece6fa5] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=33.9 K 2024-11-17T07:15:45,245 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/5d6f5d5c6eeb44db98fd6447b812fabf as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/5d6f5d5c6eeb44db98fd6447b812fabf 2024-11-17T07:15:45,245 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 24311c476f0143e4939a83eb012eac7d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731827741739 2024-11-17T07:15:45,246 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a60c2d99f0e4a7a97b74c855844242b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731827742047 2024-11-17T07:15:45,246 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 40e0100017d94b9082fd6b42cece6fa5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731827742781 2024-11-17T07:15:45,249 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/A of cab1e460ee74e5eada90871d0516a7f5 into 5d6f5d5c6eeb44db98fd6447b812fabf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:45,249 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:45,249 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/A, priority=13, startTime=1731827744805; duration=0sec 2024-11-17T07:15:45,249 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:45,249 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:A 2024-11-17T07:15:45,250 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/c7fb406aef8641d0881a6e945c7847a1 2024-11-17T07:15:45,252 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#C#compaction#447 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:45,252 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/d89b8a47912249a3b6a731ff8f664d36 is 50, key is test_row_0/C:col10/1731827743957/Put/seqid=0 2024-11-17T07:15:45,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/80d33801423d48a8b90f4970473ff797 is 50, key is test_row_0/B:col10/1731827744165/Put/seqid=0 2024-11-17T07:15:45,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742347_1523 (size=12643) 2024-11-17T07:15:45,263 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/d89b8a47912249a3b6a731ff8f664d36 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/d89b8a47912249a3b6a731ff8f664d36 2024-11-17T07:15:45,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:45,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827805247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:45,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:45,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742348_1524 (size=12301) 2024-11-17T07:15:45,267 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/80d33801423d48a8b90f4970473ff797 2024-11-17T07:15:45,267 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/C of cab1e460ee74e5eada90871d0516a7f5 into d89b8a47912249a3b6a731ff8f664d36(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:45,267 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:45,267 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/C, priority=13, startTime=1731827744805; duration=0sec 2024-11-17T07:15:45,267 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:45,267 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:C 2024-11-17T07:15:45,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/ef8b9ccab0f343e480cbd56666d9176a is 50, key is test_row_0/C:col10/1731827744165/Put/seqid=0 2024-11-17T07:15:45,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742349_1525 (size=12301) 2024-11-17T07:15:45,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827805577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:45,582 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:45,583 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:45,677 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/ef8b9ccab0f343e480cbd56666d9176a 2024-11-17T07:15:45,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/c7fb406aef8641d0881a6e945c7847a1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/c7fb406aef8641d0881a6e945c7847a1 2024-11-17T07:15:45,684 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/c7fb406aef8641d0881a6e945c7847a1, entries=150, sequenceid=453, filesize=12.0 K 2024-11-17T07:15:45,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/80d33801423d48a8b90f4970473ff797 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/80d33801423d48a8b90f4970473ff797 2024-11-17T07:15:45,691 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/80d33801423d48a8b90f4970473ff797, entries=150, sequenceid=453, filesize=12.0 K 2024-11-17T07:15:45,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/ef8b9ccab0f343e480cbd56666d9176a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/ef8b9ccab0f343e480cbd56666d9176a 2024-11-17T07:15:45,695 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/ef8b9ccab0f343e480cbd56666d9176a, entries=150, sequenceid=453, filesize=12.0 K 2024-11-17T07:15:45,696 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for cab1e460ee74e5eada90871d0516a7f5 in 871ms, sequenceid=453, compaction requested=false 2024-11-17T07:15:45,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:45,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:45,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-17T07:15:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-17T07:15:45,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-17T07:15:45,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6370 sec 2024-11-17T07:15:45,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.6420 sec 2024-11-17T07:15:46,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:46,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-17T07:15:46,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:46,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:46,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:46,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:46,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:46,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:46,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/ab6f488128ac4b3486a2bc41feb7cb2b is 50, key is test_row_0/A:col10/1731827744916/Put/seqid=0 2024-11-17T07:15:46,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742350_1526 (size=14741) 2024-11-17T07:15:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-17T07:15:46,207 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:46,207 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-17T07:15:46,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T07:15:46,209 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:46,210 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:46,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:46,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:46,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827806216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T07:15:46,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827806327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:46,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:46,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:46,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:46,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:46,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/ab6f488128ac4b3486a2bc41feb7cb2b 2024-11-17T07:15:46,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/fd0ec8a1e4614a31947ba082d162c6c0 is 50, key is test_row_0/B:col10/1731827744916/Put/seqid=0 2024-11-17T07:15:46,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742351_1527 (size=12301) 2024-11-17T07:15:46,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56850 deadline: 1731827806507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:46,513 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:46,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:46,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T07:15:46,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56810 deadline: 1731827806526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:46,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56914 deadline: 1731827806527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:46,538 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8261 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:46,538 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:46,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56838 deadline: 1731827806537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:46,545 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in cab1e460ee74e5eada90871d0516a7f5 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:15:46,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:46,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827806547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:46,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:46,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:46,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:46,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:46,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:46,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:46,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:46,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:46,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T07:15:46,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:46,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827806867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:46,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/fd0ec8a1e4614a31947ba082d162c6c0 2024-11-17T07:15:46,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/35d993f247e74373b750ae1b2193226a is 50, key is test_row_0/C:col10/1731827744916/Put/seqid=0 2024-11-17T07:15:46,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742352_1528 (size=12301) 2024-11-17T07:15:46,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:46,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:46,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:46,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:46,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:46,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:47,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:47,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:47,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:47,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/35d993f247e74373b750ae1b2193226a 2024-11-17T07:15:47,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/ab6f488128ac4b3486a2bc41feb7cb2b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ab6f488128ac4b3486a2bc41feb7cb2b 2024-11-17T07:15:47,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ab6f488128ac4b3486a2bc41feb7cb2b, entries=200, sequenceid=468, filesize=14.4 K 2024-11-17T07:15:47,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/fd0ec8a1e4614a31947ba082d162c6c0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/fd0ec8a1e4614a31947ba082d162c6c0 2024-11-17T07:15:47,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/fd0ec8a1e4614a31947ba082d162c6c0, entries=150, sequenceid=468, filesize=12.0 K 2024-11-17T07:15:47,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/35d993f247e74373b750ae1b2193226a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/35d993f247e74373b750ae1b2193226a 2024-11-17T07:15:47,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/35d993f247e74373b750ae1b2193226a, entries=150, sequenceid=468, filesize=12.0 K 2024-11-17T07:15:47,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for cab1e460ee74e5eada90871d0516a7f5 in 1250ms, sequenceid=468, compaction requested=true 2024-11-17T07:15:47,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:47,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:47,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:47,343 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:47,343 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:47,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:47,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:47,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:47,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:47,344 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:47,344 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39685 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:47,344 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/A is initiating minor compaction (all files) 2024-11-17T07:15:47,344 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/B is initiating minor compaction (all files) 2024-11-17T07:15:47,344 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/A in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,344 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/B in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,344 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/5d6f5d5c6eeb44db98fd6447b812fabf, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/c7fb406aef8641d0881a6e945c7847a1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ab6f488128ac4b3486a2bc41feb7cb2b] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=38.8 K 2024-11-17T07:15:47,344 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/772e1a59944a4ee7b27cf75c16b11c70, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/80d33801423d48a8b90f4970473ff797, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/fd0ec8a1e4614a31947ba082d162c6c0] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=36.4 K 2024-11-17T07:15:47,344 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 772e1a59944a4ee7b27cf75c16b11c70, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731827742047 2024-11-17T07:15:47,344 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d6f5d5c6eeb44db98fd6447b812fabf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731827742047 2024-11-17T07:15:47,344 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 80d33801423d48a8b90f4970473ff797, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731827744160 2024-11-17T07:15:47,344 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7fb406aef8641d0881a6e945c7847a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731827744160 2024-11-17T07:15:47,345 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting fd0ec8a1e4614a31947ba082d162c6c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731827744897 2024-11-17T07:15:47,345 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab6f488128ac4b3486a2bc41feb7cb2b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731827744883 2024-11-17T07:15:47,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T07:15:47,351 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#A#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:47,351 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#B#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:47,352 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/c3a001cf382d44f8af201c5e38e2034c is 50, key is test_row_0/A:col10/1731827744916/Put/seqid=0 2024-11-17T07:15:47,352 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/2fcc40e91d6a46b58712aba4b6b98657 is 50, key is test_row_0/B:col10/1731827744916/Put/seqid=0 2024-11-17T07:15:47,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742353_1529 (size=12745) 2024-11-17T07:15:47,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742354_1530 (size=12745) 2024-11-17T07:15:47,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:47,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-17T07:15:47,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:47,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:47,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:47,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:47,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:47,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:47,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/69c47652d1414d9084860bacce15e321 is 50, key is test_row_0/A:col10/1731827746201/Put/seqid=0 2024-11-17T07:15:47,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742355_1531 (size=14741) 2024-11-17T07:15:47,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:47,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:47,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:47,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827807453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:47,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:47,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:47,463 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:47,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:47,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827807567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:47,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:47,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:47,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:47,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:47,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:47,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:47,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,761 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/2fcc40e91d6a46b58712aba4b6b98657 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/2fcc40e91d6a46b58712aba4b6b98657 2024-11-17T07:15:47,762 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/c3a001cf382d44f8af201c5e38e2034c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/c3a001cf382d44f8af201c5e38e2034c 2024-11-17T07:15:47,765 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/B of cab1e460ee74e5eada90871d0516a7f5 into 2fcc40e91d6a46b58712aba4b6b98657(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:47,765 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:47,765 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/B, priority=13, startTime=1731827747343; duration=0sec 2024-11-17T07:15:47,765 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:47,765 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:B 2024-11-17T07:15:47,765 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:47,766 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/A of cab1e460ee74e5eada90871d0516a7f5 into c3a001cf382d44f8af201c5e38e2034c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:47,766 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:47,766 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/A, priority=13, startTime=1731827747343; duration=0sec 2024-11-17T07:15:47,766 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:47,766 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:A 2024-11-17T07:15:47,766 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:47,766 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/C is initiating minor compaction (all files) 2024-11-17T07:15:47,766 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/C in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,766 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/d89b8a47912249a3b6a731ff8f664d36, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/ef8b9ccab0f343e480cbd56666d9176a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/35d993f247e74373b750ae1b2193226a] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=36.4 K 2024-11-17T07:15:47,766 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting d89b8a47912249a3b6a731ff8f664d36, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731827742047 2024-11-17T07:15:47,767 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting ef8b9ccab0f343e480cbd56666d9176a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=453, earliestPutTs=1731827744160 2024-11-17T07:15:47,767 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 35d993f247e74373b750ae1b2193226a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731827744897 2024-11-17T07:15:47,773 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#C#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:47,773 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/79624d4fc1034fb2a768d592ec935fcb is 50, key is test_row_0/C:col10/1731827744916/Put/seqid=0 2024-11-17T07:15:47,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742356_1532 (size=12745) 2024-11-17T07:15:47,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:47,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827807787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:47,790 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:47,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/69c47652d1414d9084860bacce15e321 2024-11-17T07:15:47,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/22d6c4d8c67443b39dec7e162dcfed95 is 50, key is test_row_0/B:col10/1731827746201/Put/seqid=0 2024-11-17T07:15:47,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742357_1533 (size=12301) 2024-11-17T07:15:47,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:47,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:47,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:47,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:48,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:48,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:48,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:48,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:48,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:48,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:48,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:48,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827808097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:48,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:48,181 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/79624d4fc1034fb2a768d592ec935fcb as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/79624d4fc1034fb2a768d592ec935fcb 2024-11-17T07:15:48,185 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/C of cab1e460ee74e5eada90871d0516a7f5 into 79624d4fc1034fb2a768d592ec935fcb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:48,186 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:48,186 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/C, priority=13, startTime=1731827747343; duration=0sec 2024-11-17T07:15:48,186 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:48,186 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:C 2024-11-17T07:15:48,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:48,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:48,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:48,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:48,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:48,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:48,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:48,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/22d6c4d8c67443b39dec7e162dcfed95 2024-11-17T07:15:48,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/af2dd0ca402d456aad5e2f80e2991c2f is 50, key is test_row_0/C:col10/1731827746201/Put/seqid=0 2024-11-17T07:15:48,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742358_1534 (size=12301) 2024-11-17T07:15:48,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/af2dd0ca402d456aad5e2f80e2991c2f 2024-11-17T07:15:48,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/69c47652d1414d9084860bacce15e321 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/69c47652d1414d9084860bacce15e321 2024-11-17T07:15:48,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/69c47652d1414d9084860bacce15e321, entries=200, sequenceid=489, filesize=14.4 K 2024-11-17T07:15:48,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/22d6c4d8c67443b39dec7e162dcfed95 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/22d6c4d8c67443b39dec7e162dcfed95 2024-11-17T07:15:48,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/22d6c4d8c67443b39dec7e162dcfed95, entries=150, sequenceid=489, filesize=12.0 K 2024-11-17T07:15:48,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/af2dd0ca402d456aad5e2f80e2991c2f as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/af2dd0ca402d456aad5e2f80e2991c2f 2024-11-17T07:15:48,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/af2dd0ca402d456aad5e2f80e2991c2f, entries=150, sequenceid=489, filesize=12.0 K 2024-11-17T07:15:48,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for cab1e460ee74e5eada90871d0516a7f5 in 847ms, sequenceid=489, compaction requested=false 2024-11-17T07:15:48,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:48,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-17T07:15:48,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:48,350 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:15:48,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:48,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:48,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:48,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:48,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:48,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:48,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/6262c4a45a024d73bdca8b03da97e884 is 50, key is test_row_0/A:col10/1731827747414/Put/seqid=0 2024-11-17T07:15:48,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742359_1535 (size=12301) 2024-11-17T07:15:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T07:15:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:48,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:48,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827748708, max timestamp=1731827748737Num uniques 859; 2024-11-17T07:15:48,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:48,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827748708, max timestamp=1731827748737Num uniques 859; 2024-11-17T07:15:48,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827748708, max timestamp=1731827748737Num uniques 859; 2024-11-17T07:15:48,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:48,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:48,761 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/6262c4a45a024d73bdca8b03da97e884 2024-11-17T07:15:48,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/53385f5cdce84fb38b328be2936d415f is 50, key is test_row_0/B:col10/1731827747414/Put/seqid=0 2024-11-17T07:15:48,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742360_1536 (size=12301) 2024-11-17T07:15:48,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:48,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827808866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:48,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:48,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827808977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:48,987 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:49,172 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/53385f5cdce84fb38b328be2936d415f 2024-11-17T07:15:49,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/e1416ed346f8493187e04ae6f550cb7e is 50, key is test_row_0/C:col10/1731827747414/Put/seqid=0 2024-11-17T07:15:49,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742361_1537 (size=12301) 2024-11-17T07:15:49,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:49,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827809197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:49,209 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:49,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827809517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:49,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:49,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:49,525 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:49,584 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/e1416ed346f8493187e04ae6f550cb7e 2024-11-17T07:15:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/6262c4a45a024d73bdca8b03da97e884 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6262c4a45a024d73bdca8b03da97e884 2024-11-17T07:15:49,591 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6262c4a45a024d73bdca8b03da97e884, entries=150, sequenceid=503, filesize=12.0 K 2024-11-17T07:15:49,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/53385f5cdce84fb38b328be2936d415f as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/53385f5cdce84fb38b328be2936d415f 2024-11-17T07:15:49,594 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/53385f5cdce84fb38b328be2936d415f, entries=150, sequenceid=503, filesize=12.0 K 2024-11-17T07:15:49,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/e1416ed346f8493187e04ae6f550cb7e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/e1416ed346f8493187e04ae6f550cb7e 2024-11-17T07:15:49,597 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/e1416ed346f8493187e04ae6f550cb7e, entries=150, sequenceid=503, filesize=12.0 K 2024-11-17T07:15:49,598 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for cab1e460ee74e5eada90871d0516a7f5 in 1248ms, sequenceid=503, compaction requested=true 2024-11-17T07:15:49,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:49,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:49,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-17T07:15:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-17T07:15:49,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-17T07:15:49,601 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3890 sec 2024-11-17T07:15:49,603 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 3.3950 sec 2024-11-17T07:15:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:50,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-17T07:15:50,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:50,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:50,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:50,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:50,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:50,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:15:50,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/83757205454c408e86182747c2719583 is 50, key is test_row_0/A:col10/1731827750034/Put/seqid=0 2024-11-17T07:15:50,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742362_1538 (size=14741) 2024-11-17T07:15:50,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:50,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827810107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:50,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:50,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:50,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827810227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:50,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:50,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:50,235 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:50,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-17T07:15:50,387 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:50,387 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:15:50,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-17T07:15:50,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T07:15:50,389 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:15:50,390 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:15:50,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:15:50,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/83757205454c408e86182747c2719583 2024-11-17T07:15:50,451 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/4265de15e3a74745be5d0f8bfca315a5 is 50, key is test_row_0/B:col10/1731827750034/Put/seqid=0 2024-11-17T07:15:50,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742363_1539 (size=12301) 2024-11-17T07:15:50,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/4265de15e3a74745be5d0f8bfca315a5 2024-11-17T07:15:50,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827810447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:50,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/284ba883aa89463d8b38e0cb6e04e804 is 50, key is test_row_0/C:col10/1731827750034/Put/seqid=0 2024-11-17T07:15:50,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742364_1540 (size=12301) 2024-11-17T07:15:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T07:15:50,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T07:15:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:50,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:50,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:50,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:50,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:50,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T07:15:50,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:50,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:50,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:50,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:50,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:50,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:50,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T07:15:50,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:50,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827810767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:50,777 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:50,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T07:15:50,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:50,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:50,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:50,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:50,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:50,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:15:50,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=540 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/284ba883aa89463d8b38e0cb6e04e804 2024-11-17T07:15:50,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/83757205454c408e86182747c2719583 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/83757205454c408e86182747c2719583 2024-11-17T07:15:50,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/83757205454c408e86182747c2719583, entries=200, sequenceid=540, filesize=14.4 K 2024-11-17T07:15:50,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/4265de15e3a74745be5d0f8bfca315a5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/4265de15e3a74745be5d0f8bfca315a5 2024-11-17T07:15:50,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/4265de15e3a74745be5d0f8bfca315a5, entries=150, sequenceid=540, filesize=12.0 K 2024-11-17T07:15:50,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/284ba883aa89463d8b38e0cb6e04e804 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/284ba883aa89463d8b38e0cb6e04e804 2024-11-17T07:15:50,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/284ba883aa89463d8b38e0cb6e04e804, entries=150, sequenceid=540, filesize=12.0 K 2024-11-17T07:15:50,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for cab1e460ee74e5eada90871d0516a7f5 in 846ms, sequenceid=540, compaction requested=true 2024-11-17T07:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:50,881 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:50,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:50,881 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:15:50,883 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54528 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:15:50,883 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/A is initiating minor compaction (all files) 2024-11-17T07:15:50,883 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/A in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:50,883 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/c3a001cf382d44f8af201c5e38e2034c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/69c47652d1414d9084860bacce15e321, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6262c4a45a024d73bdca8b03da97e884, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/83757205454c408e86182747c2719583] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=53.3 K 2024-11-17T07:15:50,883 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:15:50,883 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/B is initiating minor compaction (all files) 2024-11-17T07:15:50,883 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/B in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:50,884 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/2fcc40e91d6a46b58712aba4b6b98657, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/22d6c4d8c67443b39dec7e162dcfed95, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/53385f5cdce84fb38b328be2936d415f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/4265de15e3a74745be5d0f8bfca315a5] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=48.5 K 2024-11-17T07:15:50,884 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a001cf382d44f8af201c5e38e2034c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731827744897 2024-11-17T07:15:50,884 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fcc40e91d6a46b58712aba4b6b98657, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731827744897 2024-11-17T07:15:50,884 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69c47652d1414d9084860bacce15e321, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731827746201 2024-11-17T07:15:50,884 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 22d6c4d8c67443b39dec7e162dcfed95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731827746201 2024-11-17T07:15:50,885 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6262c4a45a024d73bdca8b03da97e884, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731827747414 2024-11-17T07:15:50,885 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 53385f5cdce84fb38b328be2936d415f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731827747414 2024-11-17T07:15:50,885 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83757205454c408e86182747c2719583, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731827748837 2024-11-17T07:15:50,885 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 4265de15e3a74745be5d0f8bfca315a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731827748846 2024-11-17T07:15:50,893 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#A#compaction#465 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:50,893 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#B#compaction#466 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:50,894 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/b2e313384fed4048a529a1265e3a8f10 is 50, key is test_row_0/A:col10/1731827750034/Put/seqid=0 2024-11-17T07:15:50,894 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/255fcd319875436eb3a88feb23eae4b6 is 50, key is test_row_0/B:col10/1731827750034/Put/seqid=0 2024-11-17T07:15:50,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742366_1542 (size=12881) 2024-11-17T07:15:50,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742365_1541 (size=12881) 2024-11-17T07:15:51,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-17T07:15:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:51,002 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:15:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:51,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:51,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/80f6d66bffb346269eb1cd9275807059 is 50, key is test_row_0/A:col10/1731827750067/Put/seqid=0 2024-11-17T07:15:51,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742367_1543 (size=12301) 2024-11-17T07:15:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T07:15:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:51,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. as already flushing 2024-11-17T07:15:51,307 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/255fcd319875436eb3a88feb23eae4b6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/255fcd319875436eb3a88feb23eae4b6 2024-11-17T07:15:51,307 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/b2e313384fed4048a529a1265e3a8f10 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/b2e313384fed4048a529a1265e3a8f10 2024-11-17T07:15:51,313 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/A of cab1e460ee74e5eada90871d0516a7f5 into b2e313384fed4048a529a1265e3a8f10(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:51,313 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:51,313 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/A, priority=12, startTime=1731827750881; duration=0sec 2024-11-17T07:15:51,313 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:51,313 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:A 2024-11-17T07:15:51,313 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/B of cab1e460ee74e5eada90871d0516a7f5 into 255fcd319875436eb3a88feb23eae4b6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:51,313 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:51,313 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:15:51,313 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/B, priority=12, startTime=1731827750881; duration=0sec 2024-11-17T07:15:51,313 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:51,313 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:B 2024-11-17T07:15:51,315 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:15:51,315 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/C is initiating minor compaction (all files) 2024-11-17T07:15:51,315 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/C in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:51,315 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/79624d4fc1034fb2a768d592ec935fcb, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/af2dd0ca402d456aad5e2f80e2991c2f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/e1416ed346f8493187e04ae6f550cb7e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/284ba883aa89463d8b38e0cb6e04e804] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=48.5 K 2024-11-17T07:15:51,315 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79624d4fc1034fb2a768d592ec935fcb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731827744897 2024-11-17T07:15:51,316 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting af2dd0ca402d456aad5e2f80e2991c2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1731827746201 2024-11-17T07:15:51,316 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1416ed346f8493187e04ae6f550cb7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1731827747414 2024-11-17T07:15:51,316 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 284ba883aa89463d8b38e0cb6e04e804, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731827748846 2024-11-17T07:15:51,322 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#C#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:51,322 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/363532b48551438b8b4f46ed8fa0e5b7 is 50, key is test_row_0/C:col10/1731827750034/Put/seqid=0 2024-11-17T07:15:51,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742368_1544 (size=12881) 2024-11-17T07:15:51,330 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/363532b48551438b8b4f46ed8fa0e5b7 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/363532b48551438b8b4f46ed8fa0e5b7 2024-11-17T07:15:51,333 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/C of cab1e460ee74e5eada90871d0516a7f5 into 363532b48551438b8b4f46ed8fa0e5b7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:51,333 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:51,333 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/C, priority=12, startTime=1731827750881; duration=0sec 2024-11-17T07:15:51,333 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:51,333 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:C 2024-11-17T07:15:51,411 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/80f6d66bffb346269eb1cd9275807059 2024-11-17T07:15:51,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/eeec480856a244159d2d6b39bc0e506c is 50, key is test_row_0/B:col10/1731827750067/Put/seqid=0 2024-11-17T07:15:51,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742369_1545 (size=12301) 2024-11-17T07:15:51,422 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/eeec480856a244159d2d6b39bc0e506c 2024-11-17T07:15:51,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/09695912b5f741048425ab94ef714220 is 50, key is test_row_0/C:col10/1731827750067/Put/seqid=0 2024-11-17T07:15:51,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827751398, max timestamp=1731827751430Num uniques 859; 2024-11-17T07:15:51,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:51,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827751398, max timestamp=1731827751430Num uniques 859; 2024-11-17T07:15:51,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827751398, max timestamp=1731827751430Num uniques 859; 2024-11-17T07:15:51,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:51,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:15:51,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742370_1546 (size=12301) 2024-11-17T07:15:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T07:15:51,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827811560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:51,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:51,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827811677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:51,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:51,838 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/09695912b5f741048425ab94ef714220 2024-11-17T07:15:51,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/80f6d66bffb346269eb1cd9275807059 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/80f6d66bffb346269eb1cd9275807059 2024-11-17T07:15:51,846 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/80f6d66bffb346269eb1cd9275807059, entries=150, sequenceid=550, filesize=12.0 K 2024-11-17T07:15:51,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/eeec480856a244159d2d6b39bc0e506c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/eeec480856a244159d2d6b39bc0e506c 2024-11-17T07:15:51,850 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/eeec480856a244159d2d6b39bc0e506c, entries=150, sequenceid=550, filesize=12.0 K 2024-11-17T07:15:51,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/09695912b5f741048425ab94ef714220 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/09695912b5f741048425ab94ef714220 2024-11-17T07:15:51,853 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/09695912b5f741048425ab94ef714220, entries=150, sequenceid=550, filesize=12.0 K 2024-11-17T07:15:51,854 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for cab1e460ee74e5eada90871d0516a7f5 in 852ms, sequenceid=550, compaction requested=false 2024-11-17T07:15:51,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:51,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:51,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-17T07:15:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-17T07:15:51,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-17T07:15:51,858 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4660 sec 2024-11-17T07:15:51,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.4710 sec 2024-11-17T07:15:51,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-17T07:15:51,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:51,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:15:51,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:51,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:15:51,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:51,891 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:15:51,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/5c836774deee41569efc6bad97e0096d is 50, key is test_row_0/A:col10/1731827751889/Put/seqid=0 2024-11-17T07:15:51,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742371_1547 (size=14741) 2024-11-17T07:15:51,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/5c836774deee41569efc6bad97e0096d 2024-11-17T07:15:51,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/2251430e4a794f1c97af504528ee7ebf is 50, key is test_row_0/B:col10/1731827751889/Put/seqid=0 2024-11-17T07:15:51,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742372_1548 (size=12301) 2024-11-17T07:15:51,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/2251430e4a794f1c97af504528ee7ebf 2024-11-17T07:15:51,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/3078206ec3f448e0ac7175ec216b46b8 is 50, key is test_row_0/C:col10/1731827751889/Put/seqid=0 2024-11-17T07:15:51,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742373_1549 (size=12301) 2024-11-17T07:15:51,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:51,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827811966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:51,974 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:52,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827812077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:52,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:15:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56906 deadline: 1731827812297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 , the old value is region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:15:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., hostname=1dd62e24b303,36419,1731827614816, seqNum=2 because the exception is null or not the one we care about 2024-11-17T07:15:52,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/3078206ec3f448e0ac7175ec216b46b8 2024-11-17T07:15:52,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/5c836774deee41569efc6bad97e0096d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/5c836774deee41569efc6bad97e0096d 2024-11-17T07:15:52,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/5c836774deee41569efc6bad97e0096d, entries=200, sequenceid=591, filesize=14.4 K 2024-11-17T07:15:52,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/2251430e4a794f1c97af504528ee7ebf as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/2251430e4a794f1c97af504528ee7ebf 2024-11-17T07:15:52,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/2251430e4a794f1c97af504528ee7ebf, entries=150, sequenceid=591, filesize=12.0 K 2024-11-17T07:15:52,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/3078206ec3f448e0ac7175ec216b46b8 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/3078206ec3f448e0ac7175ec216b46b8 2024-11-17T07:15:52,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/3078206ec3f448e0ac7175ec216b46b8, entries=150, sequenceid=591, filesize=12.0 K 2024-11-17T07:15:52,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for cab1e460ee74e5eada90871d0516a7f5 in 452ms, sequenceid=591, compaction requested=true 2024-11-17T07:15:52,343 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:52,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:15:52,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:52,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:15:52,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:52,343 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:52,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store cab1e460ee74e5eada90871d0516a7f5:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:15:52,343 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:52,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:52,344 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:52,344 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:52,344 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/A is initiating minor compaction (all files) 2024-11-17T07:15:52,344 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/B is initiating minor compaction (all files) 2024-11-17T07:15:52,344 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/A in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:52,344 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/B in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:52,344 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/255fcd319875436eb3a88feb23eae4b6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/eeec480856a244159d2d6b39bc0e506c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/2251430e4a794f1c97af504528ee7ebf] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=36.6 K 2024-11-17T07:15:52,344 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/b2e313384fed4048a529a1265e3a8f10, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/80f6d66bffb346269eb1cd9275807059, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/5c836774deee41569efc6bad97e0096d] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=39.0 K 2024-11-17T07:15:52,345 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2e313384fed4048a529a1265e3a8f10, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731827748846 2024-11-17T07:15:52,345 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 255fcd319875436eb3a88feb23eae4b6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731827748846 2024-11-17T07:15:52,345 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80f6d66bffb346269eb1cd9275807059, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1731827750067 2024-11-17T07:15:52,345 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting eeec480856a244159d2d6b39bc0e506c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1731827750067 2024-11-17T07:15:52,345 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c836774deee41569efc6bad97e0096d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731827751502 2024-11-17T07:15:52,345 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2251430e4a794f1c97af504528ee7ebf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731827751547 2024-11-17T07:15:52,352 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:52,353 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/0879673dd3004467bdd9f1685d31a586 is 50, key is test_row_0/B:col10/1731827751889/Put/seqid=0 2024-11-17T07:15:52,354 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#A#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:52,354 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/9809aa6d69924660999702ec24e21f89 is 50, key is test_row_0/A:col10/1731827751889/Put/seqid=0 2024-11-17T07:15:52,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742374_1550 (size=12983) 2024-11-17T07:15:52,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742375_1551 (size=12983) 2024-11-17T07:15:52,438 INFO [Thread-2074 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2074. 2024-11-17T07:15:52,438 DEBUG [Thread-2074 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:52,438 DEBUG [Thread-2074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:52,438 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:52,438 DEBUG [Thread-2074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:52,440 INFO [Thread-2080 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2080. 2024-11-17T07:15:52,440 DEBUG [Thread-2080 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:52,440 DEBUG [Thread-2080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:52,440 DEBUG [Thread-2080 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:52,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:52,441 INFO [Thread-2082 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2082. 2024-11-17T07:15:52,441 DEBUG [Thread-2082 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:52,441 DEBUG [Thread-2082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:52,441 DEBUG [Thread-2082 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:52,441 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:52,443 INFO [Thread-2078 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2078. 2024-11-17T07:15:52,443 DEBUG [Thread-2078 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:52,443 DEBUG [Thread-2078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:52,443 DEBUG [Thread-2078 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:52,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:52,446 INFO [Thread-2076 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2076. 2024-11-17T07:15:52,446 DEBUG [Thread-2076 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:52,446 DEBUG [Thread-2076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:52,446 DEBUG [Thread-2076 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:52,446 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-17T07:15:52,537 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:52,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:15:52,608 INFO [Thread-2063 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2063. 2024-11-17T07:15:52,608 DEBUG [Thread-2063 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:52,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:52,608 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:52,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:52,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:52,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:52,608 DEBUG [Thread-2063 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:52,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:52,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:52,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:52,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/8eef273f56db4647adc16d6f7b1ff380 is 50, key is test_row_0/A:col10/1731827751930/Put/seqid=0 2024-11-17T07:15:52,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742376_1552 (size=12301) 2024-11-17T07:15:52,763 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/0879673dd3004467bdd9f1685d31a586 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/0879673dd3004467bdd9f1685d31a586 2024-11-17T07:15:52,765 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/9809aa6d69924660999702ec24e21f89 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/9809aa6d69924660999702ec24e21f89 2024-11-17T07:15:52,767 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/B of cab1e460ee74e5eada90871d0516a7f5 into 0879673dd3004467bdd9f1685d31a586(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:52,767 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:52,767 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/B, priority=13, startTime=1731827752343; duration=0sec 2024-11-17T07:15:52,768 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:15:52,768 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:B 2024-11-17T07:15:52,768 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:15:52,769 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:15:52,769 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): cab1e460ee74e5eada90871d0516a7f5/C is initiating minor compaction (all files) 2024-11-17T07:15:52,769 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/A of cab1e460ee74e5eada90871d0516a7f5 into 9809aa6d69924660999702ec24e21f89(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:52,769 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of cab1e460ee74e5eada90871d0516a7f5/C in TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:52,769 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:52,769 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/A, priority=13, startTime=1731827752343; duration=0sec 2024-11-17T07:15:52,769 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/363532b48551438b8b4f46ed8fa0e5b7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/09695912b5f741048425ab94ef714220, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/3078206ec3f448e0ac7175ec216b46b8] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp, totalSize=36.6 K 2024-11-17T07:15:52,769 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:52,769 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:A 2024-11-17T07:15:52,769 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 363532b48551438b8b4f46ed8fa0e5b7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=540, earliestPutTs=1731827748846 2024-11-17T07:15:52,769 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 09695912b5f741048425ab94ef714220, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1731827750067 2024-11-17T07:15:52,769 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 3078206ec3f448e0ac7175ec216b46b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731827751547 2024-11-17T07:15:52,774 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): cab1e460ee74e5eada90871d0516a7f5#C#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:15:52,774 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/fbcbfc5eef78417fa4b38aa31f8a9627 is 50, key is test_row_0/C:col10/1731827751889/Put/seqid=0 2024-11-17T07:15:52,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742377_1553 (size=12983) 2024-11-17T07:15:53,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/8eef273f56db4647adc16d6f7b1ff380 2024-11-17T07:15:53,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/a12eacf7f05d4a6090f2f984a2ff86e8 is 50, key is test_row_0/B:col10/1731827751930/Put/seqid=0 2024-11-17T07:15:53,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742378_1554 (size=12301) 2024-11-17T07:15:53,180 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/fbcbfc5eef78417fa4b38aa31f8a9627 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/fbcbfc5eef78417fa4b38aa31f8a9627 2024-11-17T07:15:53,183 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in cab1e460ee74e5eada90871d0516a7f5/C of cab1e460ee74e5eada90871d0516a7f5 into fbcbfc5eef78417fa4b38aa31f8a9627(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:15:53,183 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:53,183 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., storeName=cab1e460ee74e5eada90871d0516a7f5/C, priority=13, startTime=1731827752343; duration=0sec 2024-11-17T07:15:53,183 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:15:53,183 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: cab1e460ee74e5eada90871d0516a7f5:C 2024-11-17T07:15:53,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/a12eacf7f05d4a6090f2f984a2ff86e8 2024-11-17T07:15:53,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/51273115f1774b529845a6e13b070b13 is 50, key is test_row_0/C:col10/1731827751930/Put/seqid=0 2024-11-17T07:15:53,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742379_1555 (size=12301) 2024-11-17T07:15:53,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/51273115f1774b529845a6e13b070b13 2024-11-17T07:15:53,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/8eef273f56db4647adc16d6f7b1ff380 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/8eef273f56db4647adc16d6f7b1ff380 2024-11-17T07:15:53,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/8eef273f56db4647adc16d6f7b1ff380, entries=150, sequenceid=601, filesize=12.0 K 2024-11-17T07:15:53,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/a12eacf7f05d4a6090f2f984a2ff86e8 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/a12eacf7f05d4a6090f2f984a2ff86e8 2024-11-17T07:15:53,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/a12eacf7f05d4a6090f2f984a2ff86e8, entries=150, sequenceid=601, filesize=12.0 K 2024-11-17T07:15:53,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/51273115f1774b529845a6e13b070b13 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/51273115f1774b529845a6e13b070b13 2024-11-17T07:15:53,844 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/51273115f1774b529845a6e13b070b13, entries=150, sequenceid=601, filesize=12.0 K 2024-11-17T07:15:53,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for cab1e460ee74e5eada90871d0516a7f5 in 1236ms, sequenceid=601, compaction requested=false 2024-11-17T07:15:53,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for cab1e460ee74e5eada90871d0516a7f5: 2024-11-17T07:15:56,539 INFO [Thread-2067 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2067. 2024-11-17T07:15:56,539 DEBUG [Thread-2067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:56,539 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:56,539 DEBUG [Thread-2067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:56,539 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:56,568 INFO [Thread-2069 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2069. 2024-11-17T07:15:56,568 DEBUG [Thread-2069 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:56,568 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:56,568 DEBUG [Thread-2069 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:56,568 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:56,588 INFO [Thread-2065 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2065. 2024-11-17T07:15:56,588 DEBUG [Thread-2065 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:56,588 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:56,588 DEBUG [Thread-2065 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:56,588 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:56,628 INFO [Thread-2071 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2071. 2024-11-17T07:15:56,628 DEBUG [Thread-2071 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:15:56,628 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:56,628 DEBUG [Thread-2071 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:56,628 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 252 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1076 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3227 rows 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1077 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3228 rows 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1085 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3252 rows 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1083 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3246 rows 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1082 2024-11-17T07:15:56,628 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3240 rows 2024-11-17T07:15:56,628 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T07:15:56,629 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T07:15:56,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:56,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:15:56,629 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:15:56,629 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T07:15:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T07:15:56,632 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827756632"}]},"ts":"1731827756632"} 2024-11-17T07:15:56,633 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T07:15:56,633 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T07:15:56,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T07:15:56,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cab1e460ee74e5eada90871d0516a7f5, UNASSIGN}] 2024-11-17T07:15:56,636 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cab1e460ee74e5eada90871d0516a7f5, UNASSIGN 2024-11-17T07:15:56,636 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=cab1e460ee74e5eada90871d0516a7f5, regionState=CLOSING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:56,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=cab1e460ee74e5eada90871d0516a7f5, UNASSIGN because future has completed 2024-11-17T07:15:56,638 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T07:15:56,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:15:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T07:15:56,790 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:56,790 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T07:15:56,790 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing cab1e460ee74e5eada90871d0516a7f5, disabling compactions & flushes 2024-11-17T07:15:56,790 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:56,790 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:56,790 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. after waiting 0 ms 2024-11-17T07:15:56,790 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:56,790 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing cab1e460ee74e5eada90871d0516a7f5 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-17T07:15:56,790 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=A 2024-11-17T07:15:56,790 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:56,790 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=B 2024-11-17T07:15:56,790 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:56,790 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK cab1e460ee74e5eada90871d0516a7f5, store=C 2024-11-17T07:15:56,790 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:15:56,793 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/ecfaa042623b4a3b8a18449e843e20ca is 50, key is test_row_0/A:col10/1731827756567/Put/seqid=0 2024-11-17T07:15:56,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742380_1556 (size=12301) 2024-11-17T07:15:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T07:15:57,197 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/ecfaa042623b4a3b8a18449e843e20ca 2024-11-17T07:15:57,203 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/e8ab9b9d7b3946dd9c823e3e14c57ab9 is 50, key is test_row_0/B:col10/1731827756567/Put/seqid=0 2024-11-17T07:15:57,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742381_1557 (size=12301) 2024-11-17T07:15:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T07:15:57,606 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/e8ab9b9d7b3946dd9c823e3e14c57ab9 2024-11-17T07:15:57,611 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/027b9637d44e4a15814c415d5ec53e0c is 50, key is test_row_0/C:col10/1731827756567/Put/seqid=0 2024-11-17T07:15:57,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742382_1558 (size=12301) 2024-11-17T07:15:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T07:15:58,015 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/027b9637d44e4a15814c415d5ec53e0c 2024-11-17T07:15:58,018 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/A/ecfaa042623b4a3b8a18449e843e20ca as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ecfaa042623b4a3b8a18449e843e20ca 2024-11-17T07:15:58,020 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ecfaa042623b4a3b8a18449e843e20ca, entries=150, sequenceid=611, filesize=12.0 K 2024-11-17T07:15:58,021 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/B/e8ab9b9d7b3946dd9c823e3e14c57ab9 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/e8ab9b9d7b3946dd9c823e3e14c57ab9 2024-11-17T07:15:58,023 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/e8ab9b9d7b3946dd9c823e3e14c57ab9, entries=150, sequenceid=611, filesize=12.0 K 2024-11-17T07:15:58,024 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/.tmp/C/027b9637d44e4a15814c415d5ec53e0c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/027b9637d44e4a15814c415d5ec53e0c 2024-11-17T07:15:58,027 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/027b9637d44e4a15814c415d5ec53e0c, entries=150, sequenceid=611, filesize=12.0 K 2024-11-17T07:15:58,027 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for cab1e460ee74e5eada90871d0516a7f5 in 1237ms, sequenceid=611, compaction requested=true 2024-11-17T07:15:58,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ced74b60a7b34405ab78fabdfd894c4f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6f77304f1ccc49299b7a60895f59539e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/7b5c76135d8c4a6aacdd0f4976306a3f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f6234cb3f1a341849d45c4cec64e6288, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/35ff17ba7a5640ffb377ba8f55cd5694, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/a07eeba9130a49d5bb7c4f34b867249a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/b9c3c9ae968946949e0e35987b757ef0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/7460cfa5becd499aa9c08fec815662b5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/04ee62ee56c247668fb13204816af137, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/5d6f5d5c6eeb44db98fd6447b812fabf, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/dccc937d9a8146d3bd8239676a0097f3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/c7fb406aef8641d0881a6e945c7847a1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ab6f488128ac4b3486a2bc41feb7cb2b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/c3a001cf382d44f8af201c5e38e2034c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/69c47652d1414d9084860bacce15e321, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6262c4a45a024d73bdca8b03da97e884, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/83757205454c408e86182747c2719583, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/b2e313384fed4048a529a1265e3a8f10, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/80f6d66bffb346269eb1cd9275807059, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/5c836774deee41569efc6bad97e0096d] to archive 2024-11-17T07:15:58,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:15:58,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ced74b60a7b34405ab78fabdfd894c4f to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ced74b60a7b34405ab78fabdfd894c4f 2024-11-17T07:15:58,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6f77304f1ccc49299b7a60895f59539e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6f77304f1ccc49299b7a60895f59539e 2024-11-17T07:15:58,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/7b5c76135d8c4a6aacdd0f4976306a3f to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/7b5c76135d8c4a6aacdd0f4976306a3f 2024-11-17T07:15:58,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f6234cb3f1a341849d45c4cec64e6288 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/f6234cb3f1a341849d45c4cec64e6288 2024-11-17T07:15:58,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/35ff17ba7a5640ffb377ba8f55cd5694 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/35ff17ba7a5640ffb377ba8f55cd5694 2024-11-17T07:15:58,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/a07eeba9130a49d5bb7c4f34b867249a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/a07eeba9130a49d5bb7c4f34b867249a 2024-11-17T07:15:58,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/b9c3c9ae968946949e0e35987b757ef0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/b9c3c9ae968946949e0e35987b757ef0 2024-11-17T07:15:58,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/7460cfa5becd499aa9c08fec815662b5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/7460cfa5becd499aa9c08fec815662b5 2024-11-17T07:15:58,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/04ee62ee56c247668fb13204816af137 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/04ee62ee56c247668fb13204816af137 2024-11-17T07:15:58,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/5d6f5d5c6eeb44db98fd6447b812fabf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/5d6f5d5c6eeb44db98fd6447b812fabf 2024-11-17T07:15:58,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/dccc937d9a8146d3bd8239676a0097f3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/dccc937d9a8146d3bd8239676a0097f3 2024-11-17T07:15:58,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/c7fb406aef8641d0881a6e945c7847a1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/c7fb406aef8641d0881a6e945c7847a1 2024-11-17T07:15:58,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ab6f488128ac4b3486a2bc41feb7cb2b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ab6f488128ac4b3486a2bc41feb7cb2b 2024-11-17T07:15:58,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/c3a001cf382d44f8af201c5e38e2034c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/c3a001cf382d44f8af201c5e38e2034c 2024-11-17T07:15:58,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/69c47652d1414d9084860bacce15e321 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/69c47652d1414d9084860bacce15e321 2024-11-17T07:15:58,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6262c4a45a024d73bdca8b03da97e884 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/6262c4a45a024d73bdca8b03da97e884 2024-11-17T07:15:58,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/83757205454c408e86182747c2719583 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/83757205454c408e86182747c2719583 2024-11-17T07:15:58,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/b2e313384fed4048a529a1265e3a8f10 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/b2e313384fed4048a529a1265e3a8f10 2024-11-17T07:15:58,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/80f6d66bffb346269eb1cd9275807059 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/80f6d66bffb346269eb1cd9275807059 2024-11-17T07:15:58,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/5c836774deee41569efc6bad97e0096d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/5c836774deee41569efc6bad97e0096d 2024-11-17T07:15:58,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/b895883b2c4340a294493047dd5624f1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/89826a8566e9416db24e279a47aa4d17, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c8b151778dc34c77939b35446ee51230, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/6dffd28afba04b1fa32eaa0e5c987d26, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/f2deb5ddce5640bab1dd7c506ca3f480, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/9a4e60827a984641a1c0a0c49ebcb800, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c83f32e8e9244486a98c94b858d3a0b9, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/6bc35d50f8c3462bbff799c4be8131e3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/0f7512edf70e4771b1f46640a66d400b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/772e1a59944a4ee7b27cf75c16b11c70, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/bb3c6a1bea8648b2affca38df4129263, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/80d33801423d48a8b90f4970473ff797, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/2fcc40e91d6a46b58712aba4b6b98657, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/fd0ec8a1e4614a31947ba082d162c6c0, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/22d6c4d8c67443b39dec7e162dcfed95, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/53385f5cdce84fb38b328be2936d415f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/255fcd319875436eb3a88feb23eae4b6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/4265de15e3a74745be5d0f8bfca315a5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/eeec480856a244159d2d6b39bc0e506c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/2251430e4a794f1c97af504528ee7ebf] to archive 2024-11-17T07:15:58,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:15:58,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/b895883b2c4340a294493047dd5624f1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/b895883b2c4340a294493047dd5624f1 2024-11-17T07:15:58,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/89826a8566e9416db24e279a47aa4d17 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/89826a8566e9416db24e279a47aa4d17 2024-11-17T07:15:58,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c8b151778dc34c77939b35446ee51230 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c8b151778dc34c77939b35446ee51230 2024-11-17T07:15:58,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/6dffd28afba04b1fa32eaa0e5c987d26 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/6dffd28afba04b1fa32eaa0e5c987d26 2024-11-17T07:15:58,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/f2deb5ddce5640bab1dd7c506ca3f480 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/f2deb5ddce5640bab1dd7c506ca3f480 2024-11-17T07:15:58,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/9a4e60827a984641a1c0a0c49ebcb800 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/9a4e60827a984641a1c0a0c49ebcb800 2024-11-17T07:15:58,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c83f32e8e9244486a98c94b858d3a0b9 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/c83f32e8e9244486a98c94b858d3a0b9 2024-11-17T07:15:58,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/6bc35d50f8c3462bbff799c4be8131e3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/6bc35d50f8c3462bbff799c4be8131e3 2024-11-17T07:15:58,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/0f7512edf70e4771b1f46640a66d400b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/0f7512edf70e4771b1f46640a66d400b 2024-11-17T07:15:58,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/772e1a59944a4ee7b27cf75c16b11c70 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/772e1a59944a4ee7b27cf75c16b11c70 2024-11-17T07:15:58,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/bb3c6a1bea8648b2affca38df4129263 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/bb3c6a1bea8648b2affca38df4129263 2024-11-17T07:15:58,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/80d33801423d48a8b90f4970473ff797 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/80d33801423d48a8b90f4970473ff797 2024-11-17T07:15:58,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/2fcc40e91d6a46b58712aba4b6b98657 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/2fcc40e91d6a46b58712aba4b6b98657 2024-11-17T07:15:58,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/fd0ec8a1e4614a31947ba082d162c6c0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/fd0ec8a1e4614a31947ba082d162c6c0 2024-11-17T07:15:58,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/22d6c4d8c67443b39dec7e162dcfed95 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/22d6c4d8c67443b39dec7e162dcfed95 2024-11-17T07:15:58,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/53385f5cdce84fb38b328be2936d415f to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/53385f5cdce84fb38b328be2936d415f 2024-11-17T07:15:58,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/255fcd319875436eb3a88feb23eae4b6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/255fcd319875436eb3a88feb23eae4b6 2024-11-17T07:15:58,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/4265de15e3a74745be5d0f8bfca315a5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/4265de15e3a74745be5d0f8bfca315a5 2024-11-17T07:15:58,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/eeec480856a244159d2d6b39bc0e506c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/eeec480856a244159d2d6b39bc0e506c 2024-11-17T07:15:58,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/2251430e4a794f1c97af504528ee7ebf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/2251430e4a794f1c97af504528ee7ebf 2024-11-17T07:15:58,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/d3462b79184e411496c78a6a1d0d46a7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/4091a5365f784606b243b3e6b7116c21, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/e3c96c88cf14468e938a702868e92d95, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/1c0c51dbd3434e21aa60aef12a2857c3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/8d9793e75ad5404ca7c5592f5048da88, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/bfd3cd10474d408d8f50834dbc30619a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/24311c476f0143e4939a83eb012eac7d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/2d8d8832d56549bfbbd9c06b5736bd3a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/2a60c2d99f0e4a7a97b74c855844242b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/d89b8a47912249a3b6a731ff8f664d36, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/40e0100017d94b9082fd6b42cece6fa5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/ef8b9ccab0f343e480cbd56666d9176a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/79624d4fc1034fb2a768d592ec935fcb, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/35d993f247e74373b750ae1b2193226a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/af2dd0ca402d456aad5e2f80e2991c2f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/e1416ed346f8493187e04ae6f550cb7e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/363532b48551438b8b4f46ed8fa0e5b7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/284ba883aa89463d8b38e0cb6e04e804, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/09695912b5f741048425ab94ef714220, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/3078206ec3f448e0ac7175ec216b46b8] to archive 2024-11-17T07:15:58,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:15:58,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/d3462b79184e411496c78a6a1d0d46a7 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/d3462b79184e411496c78a6a1d0d46a7 2024-11-17T07:15:58,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/4091a5365f784606b243b3e6b7116c21 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/4091a5365f784606b243b3e6b7116c21 2024-11-17T07:15:58,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/e3c96c88cf14468e938a702868e92d95 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/e3c96c88cf14468e938a702868e92d95 2024-11-17T07:15:58,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/1c0c51dbd3434e21aa60aef12a2857c3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/1c0c51dbd3434e21aa60aef12a2857c3 2024-11-17T07:15:58,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/8d9793e75ad5404ca7c5592f5048da88 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/8d9793e75ad5404ca7c5592f5048da88 2024-11-17T07:15:58,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/bfd3cd10474d408d8f50834dbc30619a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/bfd3cd10474d408d8f50834dbc30619a 2024-11-17T07:15:58,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/24311c476f0143e4939a83eb012eac7d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/24311c476f0143e4939a83eb012eac7d 2024-11-17T07:15:58,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/2d8d8832d56549bfbbd9c06b5736bd3a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/2d8d8832d56549bfbbd9c06b5736bd3a 2024-11-17T07:15:58,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/2a60c2d99f0e4a7a97b74c855844242b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/2a60c2d99f0e4a7a97b74c855844242b 2024-11-17T07:15:58,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/d89b8a47912249a3b6a731ff8f664d36 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/d89b8a47912249a3b6a731ff8f664d36 2024-11-17T07:15:58,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/40e0100017d94b9082fd6b42cece6fa5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/40e0100017d94b9082fd6b42cece6fa5 2024-11-17T07:15:58,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/ef8b9ccab0f343e480cbd56666d9176a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/ef8b9ccab0f343e480cbd56666d9176a 2024-11-17T07:15:58,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/79624d4fc1034fb2a768d592ec935fcb to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/79624d4fc1034fb2a768d592ec935fcb 2024-11-17T07:15:58,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/35d993f247e74373b750ae1b2193226a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/35d993f247e74373b750ae1b2193226a 2024-11-17T07:15:58,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/af2dd0ca402d456aad5e2f80e2991c2f to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/af2dd0ca402d456aad5e2f80e2991c2f 2024-11-17T07:15:58,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/e1416ed346f8493187e04ae6f550cb7e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/e1416ed346f8493187e04ae6f550cb7e 2024-11-17T07:15:58,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/363532b48551438b8b4f46ed8fa0e5b7 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/363532b48551438b8b4f46ed8fa0e5b7 2024-11-17T07:15:58,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/284ba883aa89463d8b38e0cb6e04e804 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/284ba883aa89463d8b38e0cb6e04e804 2024-11-17T07:15:58,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/09695912b5f741048425ab94ef714220 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/09695912b5f741048425ab94ef714220 2024-11-17T07:15:58,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/3078206ec3f448e0ac7175ec216b46b8 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/3078206ec3f448e0ac7175ec216b46b8 2024-11-17T07:15:58,081 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/recovered.edits/614.seqid, newMaxSeqId=614, maxSeqId=1 2024-11-17T07:15:58,082 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. 2024-11-17T07:15:58,082 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for cab1e460ee74e5eada90871d0516a7f5: Waiting for close lock at 1731827756790Running coprocessor pre-close hooks at 1731827756790Disabling compacts and flushes for region at 1731827756790Disabling writes for close at 1731827756790Obtaining lock to block concurrent updates at 1731827756790Preparing flush snapshotting stores in cab1e460ee74e5eada90871d0516a7f5 at 1731827756790Finished memstore snapshotting TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731827756790Flushing stores of TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5. at 1731827756791 (+1 ms)Flushing cab1e460ee74e5eada90871d0516a7f5/A: creating writer at 1731827756791Flushing cab1e460ee74e5eada90871d0516a7f5/A: appending metadata at 1731827756793 (+2 ms)Flushing cab1e460ee74e5eada90871d0516a7f5/A: closing flushed file at 1731827756793Flushing cab1e460ee74e5eada90871d0516a7f5/B: creating writer at 1731827757200 (+407 ms)Flushing cab1e460ee74e5eada90871d0516a7f5/B: appending metadata at 1731827757202 (+2 ms)Flushing cab1e460ee74e5eada90871d0516a7f5/B: closing flushed file at 1731827757202Flushing cab1e460ee74e5eada90871d0516a7f5/C: creating writer at 1731827757609 (+407 ms)Flushing cab1e460ee74e5eada90871d0516a7f5/C: appending metadata at 1731827757611 (+2 ms)Flushing cab1e460ee74e5eada90871d0516a7f5/C: closing flushed file at 1731827757611Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f93d77: reopening flushed file at 1731827758017 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@328f52e9: reopening flushed file at 1731827758020 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73da4425: reopening flushed file at 1731827758023 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for cab1e460ee74e5eada90871d0516a7f5 in 1237ms, sequenceid=611, compaction requested=true at 1731827758027 (+4 ms)Writing region close event to WAL at 1731827758079 (+52 ms)Running coprocessor post-close hooks at 1731827758082 (+3 ms)Closed at 1731827758082 2024-11-17T07:15:58,084 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:58,084 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=cab1e460ee74e5eada90871d0516a7f5, regionState=CLOSED 2024-11-17T07:15:58,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:15:58,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-17T07:15:58,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure cab1e460ee74e5eada90871d0516a7f5, server=1dd62e24b303,36419,1731827614816 in 1.4480 sec 2024-11-17T07:15:58,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-17T07:15:58,088 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=cab1e460ee74e5eada90871d0516a7f5, UNASSIGN in 1.4520 sec 2024-11-17T07:15:58,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-17T07:15:58,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4550 sec 2024-11-17T07:15:58,091 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827758091"}]},"ts":"1731827758091"} 2024-11-17T07:15:58,092 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T07:15:58,092 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T07:15:58,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4620 sec 2024-11-17T07:15:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-17T07:15:58,777 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:58,778 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T07:15:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:58,779 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T07:15:58,780 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:58,782 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:58,784 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/recovered.edits] 2024-11-17T07:15:58,785 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/8eef273f56db4647adc16d6f7b1ff380 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/8eef273f56db4647adc16d6f7b1ff380 2024-11-17T07:15:58,786 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/9809aa6d69924660999702ec24e21f89 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/9809aa6d69924660999702ec24e21f89 2024-11-17T07:15:58,787 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ecfaa042623b4a3b8a18449e843e20ca to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/A/ecfaa042623b4a3b8a18449e843e20ca 2024-11-17T07:15:58,789 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/0879673dd3004467bdd9f1685d31a586 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/0879673dd3004467bdd9f1685d31a586 2024-11-17T07:15:58,789 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/a12eacf7f05d4a6090f2f984a2ff86e8 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/a12eacf7f05d4a6090f2f984a2ff86e8 2024-11-17T07:15:58,790 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/e8ab9b9d7b3946dd9c823e3e14c57ab9 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/B/e8ab9b9d7b3946dd9c823e3e14c57ab9 2024-11-17T07:15:58,792 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/027b9637d44e4a15814c415d5ec53e0c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/027b9637d44e4a15814c415d5ec53e0c 2024-11-17T07:15:58,792 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/51273115f1774b529845a6e13b070b13 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/51273115f1774b529845a6e13b070b13 2024-11-17T07:15:58,793 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/fbcbfc5eef78417fa4b38aa31f8a9627 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/C/fbcbfc5eef78417fa4b38aa31f8a9627 2024-11-17T07:15:58,795 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/recovered.edits/614.seqid to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5/recovered.edits/614.seqid 2024-11-17T07:15:58,795 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/cab1e460ee74e5eada90871d0516a7f5 2024-11-17T07:15:58,795 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T07:15:58,797 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:58,799 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T07:15:58,800 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T07:15:58,801 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:58,801 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T07:15:58,801 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731827758801"}]},"ts":"9223372036854775807"} 2024-11-17T07:15:58,802 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T07:15:58,802 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => cab1e460ee74e5eada90871d0516a7f5, NAME => 'TestAcidGuarantees,,1731827730245.cab1e460ee74e5eada90871d0516a7f5.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T07:15:58,803 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T07:15:58,803 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731827758803"}]},"ts":"9223372036854775807"} 2024-11-17T07:15:58,804 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T07:15:58,804 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:58,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 26 msec 2024-11-17T07:15:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-17T07:15:58,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T07:15:58,887 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:15:58,897 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 252), OpenFileDescriptor=447 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=393 (was 396), ProcessCount=11 (was 11), AvailableMemoryMB=1028 (was 1085) 2024-11-17T07:15:58,906 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=393, ProcessCount=11, AvailableMemoryMB=1028 2024-11-17T07:15:58,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T07:15:58,907 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T07:15:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-17T07:15:58,908 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T07:15:58,909 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:15:58,909 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-17T07:15:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T07:15:58,909 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T07:15:58,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742383_1559 (size=1061) 2024-11-17T07:15:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T07:15:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T07:15:59,315 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650 2024-11-17T07:15:59,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742384_1560 (size=53) 2024-11-17T07:15:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T07:15:59,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:15:59,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c9fb4027559a6d665f6131d22dd7458a, disabling compactions & flushes 2024-11-17T07:15:59,720 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:15:59,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:15:59,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. after waiting 0 ms 2024-11-17T07:15:59,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:15:59,720 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:15:59,720 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c9fb4027559a6d665f6131d22dd7458a: Waiting for close lock at 1731827759720Disabling compacts and flushes for region at 1731827759720Disabling writes for close at 1731827759720Writing region close event to WAL at 1731827759720Closed at 1731827759720 2024-11-17T07:15:59,721 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T07:15:59,722 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731827759721"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731827759721"}]},"ts":"1731827759721"} 2024-11-17T07:15:59,723 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T07:15:59,724 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T07:15:59,724 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827759724"}]},"ts":"1731827759724"} 2024-11-17T07:15:59,725 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-17T07:15:59,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, ASSIGN}] 2024-11-17T07:15:59,726 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, ASSIGN 2024-11-17T07:15:59,726 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, ASSIGN; state=OFFLINE, location=1dd62e24b303,36419,1731827614816; forceNewPlan=false, retain=false 2024-11-17T07:15:59,877 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=c9fb4027559a6d665f6131d22dd7458a, regionState=OPENING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:15:59,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, ASSIGN because future has completed 2024-11-17T07:15:59,879 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:16:00,033 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:00,033 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} 2024-11-17T07:16:00,033 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:00,033 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:16:00,033 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:00,033 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:00,034 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:00,035 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:16:00,035 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9fb4027559a6d665f6131d22dd7458a columnFamilyName A 2024-11-17T07:16:00,035 DEBUG [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:00,036 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.HStore(327): Store=c9fb4027559a6d665f6131d22dd7458a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:16:00,036 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:00,037 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:16:00,037 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9fb4027559a6d665f6131d22dd7458a columnFamilyName B 2024-11-17T07:16:00,037 DEBUG [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:00,037 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.HStore(327): Store=c9fb4027559a6d665f6131d22dd7458a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:16:00,037 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:00,038 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:16:00,038 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9fb4027559a6d665f6131d22dd7458a columnFamilyName C 2024-11-17T07:16:00,038 DEBUG [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:00,038 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.HStore(327): Store=c9fb4027559a6d665f6131d22dd7458a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:16:00,038 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:00,039 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:00,039 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:00,039 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:00,040 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:00,040 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:00,041 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T07:16:00,041 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:00,043 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T07:16:00,043 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened c9fb4027559a6d665f6131d22dd7458a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67195666, jitterRate=0.0012934505939483643}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T07:16:00,043 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:00,044 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for c9fb4027559a6d665f6131d22dd7458a: Running coprocessor pre-open hook at 1731827760033Writing region info on filesystem at 1731827760033Initializing all the Stores at 1731827760034 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827760034Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827760034Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827760034Cleaning up temporary data from old regions at 1731827760040 (+6 ms)Running coprocessor post-open hooks at 1731827760043 (+3 ms)Region opened successfully at 1731827760044 (+1 ms) 2024-11-17T07:16:00,044 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., pid=144, masterSystemTime=1731827760030 2024-11-17T07:16:00,046 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:00,046 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:00,046 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=c9fb4027559a6d665f6131d22dd7458a, regionState=OPEN, openSeqNum=2, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T07:16:00,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:16:00,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-17T07:16:00,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 in 169 msec 2024-11-17T07:16:00,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-17T07:16:00,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, ASSIGN in 324 msec 2024-11-17T07:16:00,051 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T07:16:00,051 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827760051"}]},"ts":"1731827760051"} 2024-11-17T07:16:00,052 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-17T07:16:00,053 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T07:16:00,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-17T07:16:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-17T07:16:01,057 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:16:01,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,058 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:16:01,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:16:01,059 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:16:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:16:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:16:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@184580fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:01,060 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:16:01,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f917329, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,061 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:16:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:16:01,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:16:01,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44534, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:16:01,063 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T07:16:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 1dd62e24b303,36491,1731827613877 2024-11-17T07:16:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f59abaa 2024-11-17T07:16:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T07:16:01,065 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T07:16:01,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-17T07:16:01,066 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T07:16:01,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-17T07:16:01,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742385_1561 (size=1097) 2024-11-17T07:16:01,476 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-17T07:16:01,476 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-17T07:16:01,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T07:16:01,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, REOPEN/MOVE}] 2024-11-17T07:16:01,480 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, REOPEN/MOVE 2024-11-17T07:16:01,481 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c9fb4027559a6d665f6131d22dd7458a, regionState=CLOSING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:01,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, REOPEN/MOVE because future has completed 2024-11-17T07:16:01,482 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T07:16:01,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:16:01,634 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,634 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T07:16:01,634 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing c9fb4027559a6d665f6131d22dd7458a, disabling compactions & flushes 2024-11-17T07:16:01,634 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:01,634 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:01,634 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. after waiting 0 ms 2024-11-17T07:16:01,634 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:01,638 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-17T07:16:01,638 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:01,638 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for c9fb4027559a6d665f6131d22dd7458a: Waiting for close lock at 1731827761634Running coprocessor pre-close hooks at 1731827761634Disabling compacts and flushes for region at 1731827761634Disabling writes for close at 1731827761634Writing region close event to WAL at 1731827761635 (+1 ms)Running coprocessor post-close hooks at 1731827761638 (+3 ms)Closed at 1731827761638 2024-11-17T07:16:01,638 WARN [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: c9fb4027559a6d665f6131d22dd7458a to self. 2024-11-17T07:16:01,640 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,640 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c9fb4027559a6d665f6131d22dd7458a, regionState=CLOSED 2024-11-17T07:16:01,641 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:16:01,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-17T07:16:01,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 in 160 msec 2024-11-17T07:16:01,643 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, REOPEN/MOVE; state=CLOSED, location=1dd62e24b303,36419,1731827614816; forceNewPlan=false, retain=true 2024-11-17T07:16:01,794 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c9fb4027559a6d665f6131d22dd7458a, regionState=OPENING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:01,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, REOPEN/MOVE because future has completed 2024-11-17T07:16:01,796 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:16:01,949 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:01,949 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} 2024-11-17T07:16:01,950 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,950 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T07:16:01,950 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,950 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,951 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,951 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:16:01,952 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9fb4027559a6d665f6131d22dd7458a columnFamilyName A 2024-11-17T07:16:01,953 DEBUG [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:01,953 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.HStore(327): Store=c9fb4027559a6d665f6131d22dd7458a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:16:01,953 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,954 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:16:01,954 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9fb4027559a6d665f6131d22dd7458a columnFamilyName B 2024-11-17T07:16:01,954 DEBUG [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:01,954 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.HStore(327): Store=c9fb4027559a6d665f6131d22dd7458a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:16:01,955 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,955 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-17T07:16:01,955 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c9fb4027559a6d665f6131d22dd7458a columnFamilyName C 2024-11-17T07:16:01,955 DEBUG [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:01,955 INFO [StoreOpener-c9fb4027559a6d665f6131d22dd7458a-1 {}] regionserver.HStore(327): Store=c9fb4027559a6d665f6131d22dd7458a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T07:16:01,955 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:01,956 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,956 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,957 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,957 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,957 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,958 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-17T07:16:01,959 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,959 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened c9fb4027559a6d665f6131d22dd7458a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65265378, jitterRate=-0.02747008204460144}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-17T07:16:01,959 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:01,960 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for c9fb4027559a6d665f6131d22dd7458a: Running coprocessor pre-open hook at 1731827761950Writing region info on filesystem at 1731827761950Initializing all the Stores at 1731827761950Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827761951 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827761951Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731827761951Cleaning up temporary data from old regions at 1731827761957 (+6 ms)Running coprocessor post-open hooks at 1731827761959 (+2 ms)Region opened successfully at 1731827761960 (+1 ms) 2024-11-17T07:16:01,960 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., pid=149, masterSystemTime=1731827761947 2024-11-17T07:16:01,962 DEBUG [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:01,962 INFO [RS_OPEN_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:01,962 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=c9fb4027559a6d665f6131d22dd7458a, regionState=OPEN, openSeqNum=5, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:01,964 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:16:01,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-17T07:16:01,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 in 169 msec 2024-11-17T07:16:01,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-17T07:16:01,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, REOPEN/MOVE in 486 msec 2024-11-17T07:16:01,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-17T07:16:01,969 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-11-17T07:16:01,970 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 903 msec 2024-11-17T07:16:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-17T07:16:01,971 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:16:01,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:16:01,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:16:01,973 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:16:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:16:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:16:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe25837, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:16:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:16:01,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:01,974 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37098, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:16:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3f233b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,975 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:16:01,975 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:16:01,975 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:16:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:16:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:16:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:16:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:16:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:01,977 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:16:01,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@700fd0e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,978 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,978 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:16:01,978 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:16:01,978 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:16:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:16:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:16:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3540f699, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:16:01,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:16:01,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:01,979 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:16:01,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a3d6e40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,980 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:16:01,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:16:01,981 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:16:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:16:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:16:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c643d27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:16:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:16:01,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:01,982 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:16:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40973148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,982 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,983 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:16:01,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:16:01,983 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:16:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:16:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:16:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33e8af76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:16:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:16:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:01,984 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:16:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@721a9116, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:16:01,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:16:01,986 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:16:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:16:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:16:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2596d900, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:16:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:16:01,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:01,987 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37208, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:16:01,987 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@970a06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,987 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af69a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,987 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:16:01,987 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:16:01,988 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:16:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:16:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:16:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43c94fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:16:01,988 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:16:01,989 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:01,989 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:16:01,990 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1adc8d33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fd90259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,990 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:16:01,990 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:16:01,991 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:16:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:16:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:16:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e326756, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:16:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:16:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:01,992 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:16:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65dd07a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594d5ccc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,993 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:16:01,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:16:01,993 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:16:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:16:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:16:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@476692c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:16:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:16:01,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:01,995 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37258, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:16:01,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fd7b86c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 1dd62e24b303,36491,-1 for getting cluster id 2024-11-17T07:16:01,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T07:16:01,996 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd37c6a52-7167-4bf7-960c-03142c6fe8ff' 2024-11-17T07:16:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T07:16:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d37c6a52-7167-4bf7-960c-03142c6fe8ff" 2024-11-17T07:16:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44b2603a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [1dd62e24b303,36491,-1] 2024-11-17T07:16:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T07:16:01,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:01,997 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T07:16:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f37b823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T07:16:01,998 DEBUG [Thread-2496 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:16:01,998 DEBUG [Thread-2503 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:16:01,998 DEBUG [Thread-2507 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:16:01,998 DEBUG [Thread-2488 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:16:01,998 DEBUG [Thread-2494 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:16:01,998 DEBUG [Thread-2501 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:16:01,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:16:01,999 DEBUG [Thread-2490 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:16:01,999 DEBUG [Thread-2499 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:16:01,999 DEBUG [Thread-2505 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:16:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:16:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:16:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:16:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:16:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:16:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:16:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:16:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:16:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:16:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:16:01,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:16:01,999 DEBUG [Thread-2492 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T07:16:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:16:02,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:16:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:16:02,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44540, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:16:02,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:16:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:16:02,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:16:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:16:02,001 DEBUG [Thread-2505 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:16:02,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:16:02,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:16:02,001 DEBUG [Thread-2505 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:16:02,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:16:02,002 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:16:02,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:16:02,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=1dd62e24b303,36419,1731827614816, seqNum=-1] 2024-11-17T07:16:02,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T07:16:02,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:16:02,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:16:02,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:16:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-17T07:16:02,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:16:02,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:16:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T07:16:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:16:02,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:16:02,006 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:16:02,007 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T07:16:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:16:02,007 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:16:02,007 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:16:02,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:16:02,008 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:16:02,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5] 2024-11-17T07:16:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:02,011 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:16:02,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:02,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:02,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:02,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:02,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:02,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:02,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731827762018, max timestamp=1731827762031Num uniques 859; 2024-11-17T07:16:02,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T07:16:02,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731827762018, max timestamp=1731827762031Num uniques 859; 2024-11-17T07:16:02,034 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-17T07:16:02,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827762028, max timestamp=1731827762032Num uniques 859; 2024-11-17T07:16:02,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:16:02,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111702db1890f3904a7db69d40d50a99b1ab_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_1/A:col10/1731827762011/Put/seqid=0 2024-11-17T07:16:02,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742386_1562 (size=9714) 2024-11-17T07:16:02,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827822048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,055 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827822048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,056 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827822055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827822060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827822061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,062 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T07:16:02,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T07:16:02,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:02,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:02,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:02,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827822167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827822167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827822167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827822167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,170 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827822168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T07:16:02,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:02,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:02,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:02,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T07:16:02,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827822377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827822377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,379 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827822377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827822377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827822378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,450 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:02,454 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111702db1890f3904a7db69d40d50a99b1ab_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111702db1890f3904a7db69d40d50a99b1ab_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:02,454 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/fd10515686c84d4588235008ac79f59c, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:02,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/fd10515686c84d4588235008ac79f59c is 175, key is test_row_1/A:col10/1731827762011/Put/seqid=0 2024-11-17T07:16:02,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742387_1563 (size=22361) 2024-11-17T07:16:02,459 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/fd10515686c84d4588235008ac79f59c 2024-11-17T07:16:02,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T07:16:02,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:02,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:02,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:02,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/89e067db16cb47b185e529a3cb98d339 is 50, key is test_row_1/B:col10/1731827762011/Put/seqid=0 2024-11-17T07:16:02,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742388_1564 (size=9657) 2024-11-17T07:16:02,621 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T07:16:02,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:02,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:02,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:02,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T07:16:02,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827822687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827822687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827822687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827822687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:02,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827822687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:02,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T07:16:02,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:02,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:02,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:02,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/89e067db16cb47b185e529a3cb98d339 2024-11-17T07:16:02,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/a8f53e68bd624d83889f1218c526e8cf is 50, key is test_row_1/C:col10/1731827762011/Put/seqid=0 2024-11-17T07:16:02,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742389_1565 (size=9657) 2024-11-17T07:16:02,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T07:16:02,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:02,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:02,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:02,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:02,932 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T07:16:03,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T07:16:03,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:03,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:03,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:03,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:03,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:03,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T07:16:03,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827823197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:03,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827823197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:03,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827823197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:03,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:03,199 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:03,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827823197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:03,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:03,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827823197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:03,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:03,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T07:16:03,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:03,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:03,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:03,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:03,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:03,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:03,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/a8f53e68bd624d83889f1218c526e8cf 2024-11-17T07:16:03,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/fd10515686c84d4588235008ac79f59c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/fd10515686c84d4588235008ac79f59c 2024-11-17T07:16:03,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/fd10515686c84d4588235008ac79f59c, entries=100, sequenceid=15, filesize=21.8 K 2024-11-17T07:16:03,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/89e067db16cb47b185e529a3cb98d339 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/89e067db16cb47b185e529a3cb98d339 2024-11-17T07:16:03,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/89e067db16cb47b185e529a3cb98d339, entries=100, sequenceid=15, filesize=9.4 K 2024-11-17T07:16:03,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/a8f53e68bd624d83889f1218c526e8cf as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/a8f53e68bd624d83889f1218c526e8cf 2024-11-17T07:16:03,330 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/a8f53e68bd624d83889f1218c526e8cf, entries=100, sequenceid=15, filesize=9.4 K 2024-11-17T07:16:03,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for c9fb4027559a6d665f6131d22dd7458a in 1320ms, sequenceid=15, compaction requested=false 2024-11-17T07:16:03,331 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-17T07:16:03,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:03,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-17T07:16:03,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:03,385 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=131.94 KB heapSize=365.84 KB 2024-11-17T07:16:03,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:03,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:16:03,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:03,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:16:03,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:03,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:16:03,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111793ac88df29b54b5f8707e3e0aa5eb0e5_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827762054/Put/seqid=0 2024-11-17T07:16:03,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742390_1566 (size=12154) 2024-11-17T07:16:03,601 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T07:16:03,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:03,800 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111793ac88df29b54b5f8707e3e0aa5eb0e5_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111793ac88df29b54b5f8707e3e0aa5eb0e5_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:03,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/b357a066e8814b758f3824d57cdb168c, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:03,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/b357a066e8814b758f3824d57cdb168c is 175, key is test_row_0/A:col10/1731827762054/Put/seqid=0 2024-11-17T07:16:03,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742391_1567 (size=30955) 2024-11-17T07:16:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T07:16:04,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:04,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:04,220 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/b357a066e8814b758f3824d57cdb168c 2024-11-17T07:16:04,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827824217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827824218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,223 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827824219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/85f5bcdb4fda4ca1a6871b45b1969152 is 50, key is test_row_0/B:col10/1731827762054/Put/seqid=0 2024-11-17T07:16:04,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827824222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827824223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,229 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742392_1568 (size=12001) 2024-11-17T07:16:04,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827824327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827824327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,331 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827824337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827824337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827824537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827824537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,541 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,541 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827824547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827824547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,639 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/85f5bcdb4fda4ca1a6871b45b1969152 2024-11-17T07:16:04,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/409fe3bd7c9344a69e4c90ed86ae907c is 50, key is test_row_0/C:col10/1731827762054/Put/seqid=0 2024-11-17T07:16:04,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742393_1569 (size=12001) 2024-11-17T07:16:04,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827824847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827824847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827824857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827824857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:04,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:05,051 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/409fe3bd7c9344a69e4c90ed86ae907c 2024-11-17T07:16:05,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/b357a066e8814b758f3824d57cdb168c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/b357a066e8814b758f3824d57cdb168c 2024-11-17T07:16:05,058 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/b357a066e8814b758f3824d57cdb168c, entries=150, sequenceid=49, filesize=30.2 K 2024-11-17T07:16:05,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/85f5bcdb4fda4ca1a6871b45b1969152 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/85f5bcdb4fda4ca1a6871b45b1969152 2024-11-17T07:16:05,061 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/85f5bcdb4fda4ca1a6871b45b1969152, entries=150, sequenceid=49, filesize=11.7 K 2024-11-17T07:16:05,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/409fe3bd7c9344a69e4c90ed86ae907c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/409fe3bd7c9344a69e4c90ed86ae907c 2024-11-17T07:16:05,065 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/409fe3bd7c9344a69e4c90ed86ae907c, entries=150, sequenceid=49, filesize=11.7 K 2024-11-17T07:16:05,066 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~131.94 KB/135110, heapSize ~365.80 KB/374576, currentSize=60.38 KB/61830 for c9fb4027559a6d665f6131d22dd7458a in 1681ms, sequenceid=49, compaction requested=false 2024-11-17T07:16:05,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:05,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:05,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-17T07:16:05,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-17T07:16:05,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-17T07:16:05,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0610 sec 2024-11-17T07:16:05,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 3.0680 sec 2024-11-17T07:16:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:05,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-17T07:16:05,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:05,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:05,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:05,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:05,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:05,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:05,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170d40e2b5fc3745429f2083c871ad7c5d_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827765358/Put/seqid=0 2024-11-17T07:16:05,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742394_1570 (size=14594) 2024-11-17T07:16:05,379 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:05,383 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170d40e2b5fc3745429f2083c871ad7c5d_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170d40e2b5fc3745429f2083c871ad7c5d_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:05,383 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/493905d53e9a4b74a10182a7ea756f61, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:05,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/493905d53e9a4b74a10182a7ea756f61 is 175, key is test_row_0/A:col10/1731827765358/Put/seqid=0 2024-11-17T07:16:05,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:05,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827825381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,387 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:05,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:05,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827825383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:05,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:05,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742395_1571 (size=39549) 2024-11-17T07:16:05,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:05,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827825386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:05,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,390 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:05,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:05,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827825386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:05,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:05,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827825497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:05,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827825497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:05,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827825497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:05,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827825497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:05,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,502 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:05,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:05,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827825707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:05,708 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827825707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:05,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827825707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,709 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:05,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827825707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:05,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:05,790 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/493905d53e9a4b74a10182a7ea756f61 2024-11-17T07:16:05,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/8a91d9fb52b64d4787002ec955dc8fb7 is 50, key is test_row_0/B:col10/1731827765358/Put/seqid=0 2024-11-17T07:16:05,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742396_1572 (size=12001) 2024-11-17T07:16:06,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827826017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:06,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827826017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:06,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827826017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:06,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:06,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827826017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:06,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-17T07:16:06,157 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:16:06,158 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:16:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-17T07:16:06,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T07:16:06,159 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:16:06,160 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:16:06,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:16:06,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/8a91d9fb52b64d4787002ec955dc8fb7 2024-11-17T07:16:06,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/33f191f075ef4ad7aa89c2a8dfb6215b is 50, key is test_row_0/C:col10/1731827765358/Put/seqid=0 2024-11-17T07:16:06,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742397_1573 (size=12001) 2024-11-17T07:16:06,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:06,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827826237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:06,240 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:06,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T07:16:06,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T07:16:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:06,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:06,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:06,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:06,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:06,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T07:16:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:06,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:06,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:06,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:06,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T07:16:06,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827826527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:06,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827826527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,529 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:06,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827826527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:06,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:06,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827826528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:06,530 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:06,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/33f191f075ef4ad7aa89c2a8dfb6215b 2024-11-17T07:16:06,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T07:16:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:06,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:06,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:06,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:06,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/493905d53e9a4b74a10182a7ea756f61 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/493905d53e9a4b74a10182a7ea756f61 2024-11-17T07:16:06,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/493905d53e9a4b74a10182a7ea756f61, entries=200, sequenceid=62, filesize=38.6 K 2024-11-17T07:16:06,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/8a91d9fb52b64d4787002ec955dc8fb7 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/8a91d9fb52b64d4787002ec955dc8fb7 2024-11-17T07:16:06,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/8a91d9fb52b64d4787002ec955dc8fb7, entries=150, sequenceid=62, filesize=11.7 K 2024-11-17T07:16:06,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/33f191f075ef4ad7aa89c2a8dfb6215b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/33f191f075ef4ad7aa89c2a8dfb6215b 2024-11-17T07:16:06,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/33f191f075ef4ad7aa89c2a8dfb6215b, entries=150, sequenceid=62, filesize=11.7 K 2024-11-17T07:16:06,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c9fb4027559a6d665f6131d22dd7458a in 1274ms, sequenceid=62, compaction requested=true 2024-11-17T07:16:06,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:06,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:16:06,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:06,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:16:06,633 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:06,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:06,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:16:06,633 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:06,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:06,634 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:06,634 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:06,634 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/B is initiating minor compaction (all files) 2024-11-17T07:16:06,634 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/B in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:06,634 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/89e067db16cb47b185e529a3cb98d339, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/85f5bcdb4fda4ca1a6871b45b1969152, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/8a91d9fb52b64d4787002ec955dc8fb7] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=32.9 K 2024-11-17T07:16:06,634 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/A is initiating minor compaction (all files) 2024-11-17T07:16:06,634 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/A in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:06,634 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/fd10515686c84d4588235008ac79f59c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/b357a066e8814b758f3824d57cdb168c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/493905d53e9a4b74a10182a7ea756f61] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=90.7 K 2024-11-17T07:16:06,634 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:06,634 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 89e067db16cb47b185e529a3cb98d339, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731827762009 2024-11-17T07:16:06,634 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/fd10515686c84d4588235008ac79f59c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/b357a066e8814b758f3824d57cdb168c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/493905d53e9a4b74a10182a7ea756f61] 2024-11-17T07:16:06,635 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 85f5bcdb4fda4ca1a6871b45b1969152, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731827762046 2024-11-17T07:16:06,635 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd10515686c84d4588235008ac79f59c, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731827762009 2024-11-17T07:16:06,635 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting b357a066e8814b758f3824d57cdb168c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731827762046 2024-11-17T07:16:06,635 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a91d9fb52b64d4787002ec955dc8fb7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731827764218 2024-11-17T07:16:06,635 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 493905d53e9a4b74a10182a7ea756f61, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731827764218 2024-11-17T07:16:06,640 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:06,641 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#B#compaction#492 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:06,641 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/76bc91ca13614d76bc4712eae4f37882 is 50, key is test_row_0/B:col10/1731827765358/Put/seqid=0 2024-11-17T07:16:06,644 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117dd19a2316a6f428386c6ef7f91e4535d_c9fb4027559a6d665f6131d22dd7458a store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:06,646 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117dd19a2316a6f428386c6ef7f91e4535d_c9fb4027559a6d665f6131d22dd7458a, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:06,646 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117dd19a2316a6f428386c6ef7f91e4535d_c9fb4027559a6d665f6131d22dd7458a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:06,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742398_1574 (size=12104) 2024-11-17T07:16:06,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742399_1575 (size=4469) 2024-11-17T07:16:06,650 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#A#compaction#493 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:06,650 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/374522f116f047a4a6a4b833921c961d is 175, key is test_row_0/A:col10/1731827765358/Put/seqid=0 2024-11-17T07:16:06,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742400_1576 (size=31058) 2024-11-17T07:16:06,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-17T07:16:06,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:06,772 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-17T07:16:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:06,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111757afe2cf83f34ae8b795f1bca60b9b46_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827765376/Put/seqid=0 2024-11-17T07:16:06,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742401_1577 (size=12154) 2024-11-17T07:16:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T07:16:06,789 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111757afe2cf83f34ae8b795f1bca60b9b46_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111757afe2cf83f34ae8b795f1bca60b9b46_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:06,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/23b2256d362642f2b5c8da2503586f5e, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:06,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/23b2256d362642f2b5c8da2503586f5e is 175, key is test_row_0/A:col10/1731827765376/Put/seqid=0 2024-11-17T07:16:06,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742402_1578 (size=30955) 2024-11-17T07:16:07,051 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/76bc91ca13614d76bc4712eae4f37882 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/76bc91ca13614d76bc4712eae4f37882 2024-11-17T07:16:07,055 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/B of c9fb4027559a6d665f6131d22dd7458a into 76bc91ca13614d76bc4712eae4f37882(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:07,055 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:07,055 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/B, priority=13, startTime=1731827766633; duration=0sec 2024-11-17T07:16:07,056 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:07,056 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:B 2024-11-17T07:16:07,056 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:07,057 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:07,057 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/C is initiating minor compaction (all files) 2024-11-17T07:16:07,057 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/C in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:07,057 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/a8f53e68bd624d83889f1218c526e8cf, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/409fe3bd7c9344a69e4c90ed86ae907c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/33f191f075ef4ad7aa89c2a8dfb6215b] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=32.9 K 2024-11-17T07:16:07,057 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting a8f53e68bd624d83889f1218c526e8cf, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731827762009 2024-11-17T07:16:07,058 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 409fe3bd7c9344a69e4c90ed86ae907c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731827762046 2024-11-17T07:16:07,058 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/374522f116f047a4a6a4b833921c961d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/374522f116f047a4a6a4b833921c961d 2024-11-17T07:16:07,058 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 33f191f075ef4ad7aa89c2a8dfb6215b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731827764218 2024-11-17T07:16:07,062 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/A of c9fb4027559a6d665f6131d22dd7458a into 374522f116f047a4a6a4b833921c961d(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:07,062 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:07,062 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/A, priority=13, startTime=1731827766633; duration=0sec 2024-11-17T07:16:07,062 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:07,062 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:A 2024-11-17T07:16:07,064 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#C#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:07,064 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/944eedc463764249a15aa438b5c17af7 is 50, key is test_row_0/C:col10/1731827765358/Put/seqid=0 2024-11-17T07:16:07,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742403_1579 (size=12104) 2024-11-17T07:16:07,200 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/23b2256d362642f2b5c8da2503586f5e 2024-11-17T07:16:07,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/b612dba8b6554817b59bf6b4a61bbc4d is 50, key is test_row_0/B:col10/1731827765376/Put/seqid=0 2024-11-17T07:16:07,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742404_1580 (size=12001) 2024-11-17T07:16:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T07:16:07,472 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/944eedc463764249a15aa438b5c17af7 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/944eedc463764249a15aa438b5c17af7 2024-11-17T07:16:07,477 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/C of c9fb4027559a6d665f6131d22dd7458a into 944eedc463764249a15aa438b5c17af7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:07,477 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:07,477 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/C, priority=13, startTime=1731827766633; duration=0sec 2024-11-17T07:16:07,477 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:07,477 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:C 2024-11-17T07:16:07,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:07,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:07,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:07,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827827550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:07,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:07,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827827551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:07,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827827552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:07,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827827553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:07,617 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/b612dba8b6554817b59bf6b4a61bbc4d 2024-11-17T07:16:07,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/461448d2b3ea40989e144dc07a8d36c2 is 50, key is test_row_0/C:col10/1731827765376/Put/seqid=0 2024-11-17T07:16:07,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742405_1581 (size=12001) 2024-11-17T07:16:07,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:07,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827827657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:07,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:07,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827827667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827827667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:07,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827827668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:07,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:07,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827827867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:07,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:07,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:07,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827827877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:07,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827827877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:07,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827827878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:07,881 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,028 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/461448d2b3ea40989e144dc07a8d36c2 2024-11-17T07:16:08,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/23b2256d362642f2b5c8da2503586f5e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/23b2256d362642f2b5c8da2503586f5e 2024-11-17T07:16:08,035 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/23b2256d362642f2b5c8da2503586f5e, entries=150, sequenceid=81, filesize=30.2 K 2024-11-17T07:16:08,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/b612dba8b6554817b59bf6b4a61bbc4d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/b612dba8b6554817b59bf6b4a61bbc4d 2024-11-17T07:16:08,039 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/b612dba8b6554817b59bf6b4a61bbc4d, entries=150, sequenceid=81, filesize=11.7 K 2024-11-17T07:16:08,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/461448d2b3ea40989e144dc07a8d36c2 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/461448d2b3ea40989e144dc07a8d36c2 2024-11-17T07:16:08,042 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/461448d2b3ea40989e144dc07a8d36c2, entries=150, sequenceid=81, filesize=11.7 K 2024-11-17T07:16:08,043 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c9fb4027559a6d665f6131d22dd7458a in 1271ms, sequenceid=81, compaction requested=false 2024-11-17T07:16:08,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:08,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:08,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-17T07:16:08,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-17T07:16:08,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-17T07:16:08,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8840 sec 2024-11-17T07:16:08,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.8890 sec 2024-11-17T07:16:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:08,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-17T07:16:08,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:08,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:08,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:08,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:08,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:08,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:08,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176e5348aff75b4ac982f71be12f6ed880_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827767550/Put/seqid=0 2024-11-17T07:16:08,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742406_1582 (size=14594) 2024-11-17T07:16:08,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827828202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827828206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827828207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827828208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-17T07:16:08,308 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:16:08,308 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:16:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-17T07:16:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T07:16:08,310 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:16:08,311 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:16:08,311 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:16:08,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827828316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827828317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827828317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827828317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T07:16:08,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T07:16:08,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:08,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:08,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:08,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:08,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:08,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827828527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827828527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827828527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827828527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,531 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,590 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:08,594 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411176e5348aff75b4ac982f71be12f6ed880_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176e5348aff75b4ac982f71be12f6ed880_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:08,594 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/63090727187a461eaff7076db6c08b5c, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:08,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/63090727187a461eaff7076db6c08b5c is 175, key is test_row_0/A:col10/1731827767550/Put/seqid=0 2024-11-17T07:16:08,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742407_1583 (size=39549) 2024-11-17T07:16:08,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T07:16:08,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:08,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:08,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:08,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:08,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:08,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T07:16:08,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T07:16:08,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:08,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:08,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:08,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:08,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:08,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:08,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827828837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827828838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827828838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,842 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:08,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827828838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:08,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:08,842 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:08,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T07:16:08,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:08,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:08,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:08,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:08,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:08,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T07:16:09,008 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/63090727187a461eaff7076db6c08b5c 2024-11-17T07:16:09,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/0fac1e31df4441b8966d7f15ea5006cc is 50, key is test_row_0/B:col10/1731827767550/Put/seqid=0 2024-11-17T07:16:09,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742408_1584 (size=12001) 2024-11-17T07:16:09,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T07:16:09,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:09,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T07:16:09,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:09,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827829347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:09,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827829347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:09,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:09,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:09,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:09,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:09,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:09,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827829357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:09,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827829357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:09,361 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:09,362 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:09,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T07:16:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:09,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,419 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/0fac1e31df4441b8966d7f15ea5006cc 2024-11-17T07:16:09,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/bfb92f13c7494ecea9cea68570ca656a is 50, key is test_row_0/C:col10/1731827767550/Put/seqid=0 2024-11-17T07:16:09,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742409_1585 (size=12001) 2024-11-17T07:16:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T07:16:09,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T07:16:09,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:09,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T07:16:09,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:09,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/bfb92f13c7494ecea9cea68570ca656a 2024-11-17T07:16:09,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/63090727187a461eaff7076db6c08b5c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/63090727187a461eaff7076db6c08b5c 2024-11-17T07:16:09,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T07:16:09,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:09,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,838 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/63090727187a461eaff7076db6c08b5c, entries=200, sequenceid=98, filesize=38.6 K 2024-11-17T07:16:09,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:09,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/0fac1e31df4441b8966d7f15ea5006cc as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/0fac1e31df4441b8966d7f15ea5006cc 2024-11-17T07:16:09,842 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/0fac1e31df4441b8966d7f15ea5006cc, entries=150, sequenceid=98, filesize=11.7 K 2024-11-17T07:16:09,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/bfb92f13c7494ecea9cea68570ca656a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/bfb92f13c7494ecea9cea68570ca656a 2024-11-17T07:16:09,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/bfb92f13c7494ecea9cea68570ca656a, entries=150, sequenceid=98, filesize=11.7 K 2024-11-17T07:16:09,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c9fb4027559a6d665f6131d22dd7458a in 1667ms, sequenceid=98, compaction requested=true 2024-11-17T07:16:09,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:16:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:16:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:09,847 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:16:09,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:09,847 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:09,847 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:09,848 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:09,848 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/B is initiating minor compaction (all files) 2024-11-17T07:16:09,848 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/A is initiating minor compaction (all files) 2024-11-17T07:16:09,848 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/B in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,848 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/A in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,848 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/76bc91ca13614d76bc4712eae4f37882, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/b612dba8b6554817b59bf6b4a61bbc4d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/0fac1e31df4441b8966d7f15ea5006cc] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=35.3 K 2024-11-17T07:16:09,848 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/374522f116f047a4a6a4b833921c961d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/23b2256d362642f2b5c8da2503586f5e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/63090727187a461eaff7076db6c08b5c] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=99.2 K 2024-11-17T07:16:09,848 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,848 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/374522f116f047a4a6a4b833921c961d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/23b2256d362642f2b5c8da2503586f5e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/63090727187a461eaff7076db6c08b5c] 2024-11-17T07:16:09,848 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 76bc91ca13614d76bc4712eae4f37882, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731827764218 2024-11-17T07:16:09,848 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 374522f116f047a4a6a4b833921c961d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731827764218 2024-11-17T07:16:09,849 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting b612dba8b6554817b59bf6b4a61bbc4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731827765376 2024-11-17T07:16:09,849 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23b2256d362642f2b5c8da2503586f5e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731827765376 2024-11-17T07:16:09,849 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fac1e31df4441b8966d7f15ea5006cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731827767550 2024-11-17T07:16:09,849 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63090727187a461eaff7076db6c08b5c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731827767544 2024-11-17T07:16:09,855 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:09,856 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#B#compaction#501 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:09,856 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/3b87f1432c304d4384448a4f75e09743 is 50, key is test_row_0/B:col10/1731827767550/Put/seqid=0 2024-11-17T07:16:09,857 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117d66efc78e47c4deb9aad736fb5d072c1_c9fb4027559a6d665f6131d22dd7458a store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:09,859 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117d66efc78e47c4deb9aad736fb5d072c1_c9fb4027559a6d665f6131d22dd7458a, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:09,859 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d66efc78e47c4deb9aad736fb5d072c1_c9fb4027559a6d665f6131d22dd7458a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:09,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742410_1586 (size=12207) 2024-11-17T07:16:09,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742411_1587 (size=4469) 2024-11-17T07:16:09,865 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/3b87f1432c304d4384448a4f75e09743 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/3b87f1432c304d4384448a4f75e09743 2024-11-17T07:16:09,869 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/B of c9fb4027559a6d665f6131d22dd7458a into 3b87f1432c304d4384448a4f75e09743(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:09,869 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:09,869 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/B, priority=13, startTime=1731827769847; duration=0sec 2024-11-17T07:16:09,869 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:09,869 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:B 2024-11-17T07:16:09,869 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:09,870 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:09,870 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/C is initiating minor compaction (all files) 2024-11-17T07:16:09,870 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/C in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,870 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/944eedc463764249a15aa438b5c17af7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/461448d2b3ea40989e144dc07a8d36c2, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/bfb92f13c7494ecea9cea68570ca656a] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=35.3 K 2024-11-17T07:16:09,870 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 944eedc463764249a15aa438b5c17af7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731827764218 2024-11-17T07:16:09,870 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 461448d2b3ea40989e144dc07a8d36c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731827765376 2024-11-17T07:16:09,871 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting bfb92f13c7494ecea9cea68570ca656a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731827767550 2024-11-17T07:16:09,879 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#C#compaction#503 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:09,879 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/66a5d000f528424e82932daafc4c4e15 is 50, key is test_row_0/C:col10/1731827767550/Put/seqid=0 2024-11-17T07:16:09,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742412_1588 (size=12207) 2024-11-17T07:16:09,899 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/66a5d000f528424e82932daafc4c4e15 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/66a5d000f528424e82932daafc4c4e15 2024-11-17T07:16:09,903 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/C of c9fb4027559a6d665f6131d22dd7458a into 66a5d000f528424e82932daafc4c4e15(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:09,903 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:09,903 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/C, priority=13, startTime=1731827769847; duration=0sec 2024-11-17T07:16:09,903 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:09,903 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:C 2024-11-17T07:16:09,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-17T07:16:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:09,991 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-17T07:16:09,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:09,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:09,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:09,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:09,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:09,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:09,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179beaf38d10954eb68f40508383b17ce7_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827768207/Put/seqid=0 2024-11-17T07:16:10,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742413_1589 (size=12154) 2024-11-17T07:16:10,264 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#A#compaction#502 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:10,264 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/6a3e3b159a444f22ba67cde20d704aa5 is 175, key is test_row_0/A:col10/1731827767550/Put/seqid=0 2024-11-17T07:16:10,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742414_1590 (size=31161) 2024-11-17T07:16:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:10,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:10,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827830314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:10,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827830357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:10,360 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,361 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:10,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827830367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:10,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827830367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:10,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:10,371 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827830367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:10,371 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:10,372 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:10,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:10,405 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411179beaf38d10954eb68f40508383b17ce7_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179beaf38d10954eb68f40508383b17ce7_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:10,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/2c6094000194435eaf455f29266c8b4e, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:10,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/2c6094000194435eaf455f29266c8b4e is 175, key is test_row_0/A:col10/1731827768207/Put/seqid=0 2024-11-17T07:16:10,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742415_1591 (size=30955) 2024-11-17T07:16:10,410 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/2c6094000194435eaf455f29266c8b4e 2024-11-17T07:16:10,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/29e9bcbebd9849a3ab6f6997202d1555 is 50, key is test_row_0/B:col10/1731827768207/Put/seqid=0 2024-11-17T07:16:10,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742416_1592 (size=12001) 2024-11-17T07:16:10,430 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827830427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,431 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T07:16:10,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827830637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:10,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:10,673 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/6a3e3b159a444f22ba67cde20d704aa5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/6a3e3b159a444f22ba67cde20d704aa5 2024-11-17T07:16:10,677 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/A of c9fb4027559a6d665f6131d22dd7458a into 6a3e3b159a444f22ba67cde20d704aa5(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:10,677 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:10,677 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/A, priority=13, startTime=1731827769846; duration=0sec 2024-11-17T07:16:10,677 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:10,677 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:A 2024-11-17T07:16:10,828 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/29e9bcbebd9849a3ab6f6997202d1555 2024-11-17T07:16:10,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/bd7421628b3c47a29897e6a3048c5c09 is 50, key is test_row_0/C:col10/1731827768207/Put/seqid=0 2024-11-17T07:16:10,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742417_1593 (size=12001) 2024-11-17T07:16:10,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827830947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:10,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:10,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:11,238 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/bd7421628b3c47a29897e6a3048c5c09 2024-11-17T07:16:11,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/2c6094000194435eaf455f29266c8b4e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/2c6094000194435eaf455f29266c8b4e 2024-11-17T07:16:11,245 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/2c6094000194435eaf455f29266c8b4e, entries=150, sequenceid=118, filesize=30.2 K 2024-11-17T07:16:11,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/29e9bcbebd9849a3ab6f6997202d1555 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/29e9bcbebd9849a3ab6f6997202d1555 2024-11-17T07:16:11,249 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/29e9bcbebd9849a3ab6f6997202d1555, entries=150, sequenceid=118, filesize=11.7 K 2024-11-17T07:16:11,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/bd7421628b3c47a29897e6a3048c5c09 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/bd7421628b3c47a29897e6a3048c5c09 2024-11-17T07:16:11,252 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/bd7421628b3c47a29897e6a3048c5c09, entries=150, sequenceid=118, filesize=11.7 K 2024-11-17T07:16:11,254 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c9fb4027559a6d665f6131d22dd7458a in 1263ms, sequenceid=118, compaction requested=false 2024-11-17T07:16:11,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:11,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:11,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-17T07:16:11,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-17T07:16:11,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-17T07:16:11,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9440 sec 2024-11-17T07:16:11,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.9490 sec 2024-11-17T07:16:11,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:11,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-17T07:16:11,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:11,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:11,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:11,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:11,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:11,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:11,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117016ef61065544c23ba3137d022f83dfb_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827770275/Put/seqid=0 2024-11-17T07:16:11,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742418_1594 (size=14744) 2024-11-17T07:16:11,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:11,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827831521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:11,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:11,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827831627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:11,630 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:11,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827831837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:11,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:11,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:11,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:11,871 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:11,874 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117016ef61065544c23ba3137d022f83dfb_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117016ef61065544c23ba3137d022f83dfb_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:11,875 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/39dcaccc8c5b4241b6848cb4b1451c47, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:11,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/39dcaccc8c5b4241b6848cb4b1451c47 is 175, key is test_row_0/A:col10/1731827770275/Put/seqid=0 2024-11-17T07:16:11,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742419_1595 (size=39699) 2024-11-17T07:16:11,881 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/39dcaccc8c5b4241b6848cb4b1451c47 2024-11-17T07:16:11,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/e6b2305539344f9a8a93b557235ed89a is 50, key is test_row_0/B:col10/1731827770275/Put/seqid=0 2024-11-17T07:16:11,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742420_1596 (size=12101) 2024-11-17T07:16:12,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:12,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827832147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:12,150 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:12,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/e6b2305539344f9a8a93b557235ed89a 2024-11-17T07:16:12,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/f5376762a41d452d97505f704953bcf4 is 50, key is test_row_0/C:col10/1731827770275/Put/seqid=0 2024-11-17T07:16:12,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742421_1597 (size=12101) 2024-11-17T07:16:12,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:12,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827832367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:12,370 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:12,370 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:12,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827832377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:12,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827832377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:12,381 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:12,381 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827832387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:12,390 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-17T07:16:12,478 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:16:12,478 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:16:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-17T07:16:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T07:16:12,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:16:12,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:16:12,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:16:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T07:16:12,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T07:16:12,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:12,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:12,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:12,633 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:12,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:12,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827832657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:12,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:12,660 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:12,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/f5376762a41d452d97505f704953bcf4 2024-11-17T07:16:12,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/39dcaccc8c5b4241b6848cb4b1451c47 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/39dcaccc8c5b4241b6848cb4b1451c47 2024-11-17T07:16:12,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/39dcaccc8c5b4241b6848cb4b1451c47, entries=200, sequenceid=134, filesize=38.8 K 2024-11-17T07:16:12,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/e6b2305539344f9a8a93b557235ed89a as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e6b2305539344f9a8a93b557235ed89a 2024-11-17T07:16:12,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e6b2305539344f9a8a93b557235ed89a, entries=150, sequenceid=134, filesize=11.8 K 2024-11-17T07:16:12,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/f5376762a41d452d97505f704953bcf4 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/f5376762a41d452d97505f704953bcf4 2024-11-17T07:16:12,715 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/f5376762a41d452d97505f704953bcf4, entries=150, sequenceid=134, filesize=11.8 K 2024-11-17T07:16:12,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c9fb4027559a6d665f6131d22dd7458a in 1256ms, sequenceid=134, compaction requested=true 2024-11-17T07:16:12,716 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:12,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:16:12,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:12,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:16:12,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:12,716 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:12,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:16:12,716 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:12,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:12,717 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:12,717 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:12,717 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/B is initiating minor compaction (all files) 2024-11-17T07:16:12,717 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/A is initiating minor compaction (all files) 2024-11-17T07:16:12,717 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/A in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:12,717 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/B in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:12,717 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/6a3e3b159a444f22ba67cde20d704aa5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/2c6094000194435eaf455f29266c8b4e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/39dcaccc8c5b4241b6848cb4b1451c47] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=99.4 K 2024-11-17T07:16:12,717 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/3b87f1432c304d4384448a4f75e09743, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/29e9bcbebd9849a3ab6f6997202d1555, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e6b2305539344f9a8a93b557235ed89a] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=35.5 K 2024-11-17T07:16:12,717 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:12,717 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/6a3e3b159a444f22ba67cde20d704aa5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/2c6094000194435eaf455f29266c8b4e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/39dcaccc8c5b4241b6848cb4b1451c47] 2024-11-17T07:16:12,718 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b87f1432c304d4384448a4f75e09743, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731827767550 2024-11-17T07:16:12,718 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a3e3b159a444f22ba67cde20d704aa5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731827767550 2024-11-17T07:16:12,718 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 29e9bcbebd9849a3ab6f6997202d1555, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731827768193 2024-11-17T07:16:12,718 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting e6b2305539344f9a8a93b557235ed89a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731827770275 2024-11-17T07:16:12,718 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c6094000194435eaf455f29266c8b4e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731827768193 2024-11-17T07:16:12,719 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39dcaccc8c5b4241b6848cb4b1451c47, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731827770275 2024-11-17T07:16:12,727 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:12,728 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#B#compaction#511 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:12,728 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117d914e2489b3c4b5bab544cb33b6705d8_c9fb4027559a6d665f6131d22dd7458a store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:12,729 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/ca65211f010045b1a4125f93e3a1615b is 50, key is test_row_0/B:col10/1731827770275/Put/seqid=0 2024-11-17T07:16:12,731 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117d914e2489b3c4b5bab544cb33b6705d8_c9fb4027559a6d665f6131d22dd7458a, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:12,731 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d914e2489b3c4b5bab544cb33b6705d8_c9fb4027559a6d665f6131d22dd7458a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:12,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742422_1598 (size=12409) 2024-11-17T07:16:12,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742423_1599 (size=4469) 2024-11-17T07:16:12,748 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#A#compaction#510 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:12,749 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/2516f1ddb1824665bc194d6186e97a06 is 175, key is test_row_0/A:col10/1731827770275/Put/seqid=0 2024-11-17T07:16:12,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742424_1600 (size=31363) 2024-11-17T07:16:12,772 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/2516f1ddb1824665bc194d6186e97a06 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/2516f1ddb1824665bc194d6186e97a06 2024-11-17T07:16:12,779 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/A of c9fb4027559a6d665f6131d22dd7458a into 2516f1ddb1824665bc194d6186e97a06(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:12,779 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:12,779 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/A, priority=13, startTime=1731827772716; duration=0sec 2024-11-17T07:16:12,779 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:12,779 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:A 2024-11-17T07:16:12,779 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:12,781 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:12,781 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/C is initiating minor compaction (all files) 2024-11-17T07:16:12,781 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/C in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:12,781 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/66a5d000f528424e82932daafc4c4e15, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/bd7421628b3c47a29897e6a3048c5c09, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/f5376762a41d452d97505f704953bcf4] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=35.5 K 2024-11-17T07:16:12,781 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66a5d000f528424e82932daafc4c4e15, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731827767550 2024-11-17T07:16:12,783 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd7421628b3c47a29897e6a3048c5c09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731827768193 2024-11-17T07:16:12,783 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5376762a41d452d97505f704953bcf4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731827770275 2024-11-17T07:16:12,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-17T07:16:12,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:12,786 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-17T07:16:12,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:12,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:12,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:12,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:12,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:12,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:12,791 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#C#compaction#512 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:12,792 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/cc2f8967d71347299503a7a3df4310d3 is 50, key is test_row_0/C:col10/1731827770275/Put/seqid=0 2024-11-17T07:16:12,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117628072ddb9bf4af489f68a8ef84104a6_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827771520/Put/seqid=0 2024-11-17T07:16:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T07:16:12,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742425_1601 (size=12409) 2024-11-17T07:16:12,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742426_1602 (size=12304) 2024-11-17T07:16:12,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:12,804 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117628072ddb9bf4af489f68a8ef84104a6_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117628072ddb9bf4af489f68a8ef84104a6_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:12,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/40ff131254044125b58783399ed76bc6, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:12,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/40ff131254044125b58783399ed76bc6 is 175, key is test_row_0/A:col10/1731827771520/Put/seqid=0 2024-11-17T07:16:12,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742427_1603 (size=31105) 2024-11-17T07:16:12,811 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/40ff131254044125b58783399ed76bc6 2024-11-17T07:16:12,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/54830c508d144e59940b86ec8e11cf7c is 50, key is test_row_0/B:col10/1731827771520/Put/seqid=0 2024-11-17T07:16:12,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742428_1604 (size=12151) 2024-11-17T07:16:12,835 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/54830c508d144e59940b86ec8e11cf7c 2024-11-17T07:16:12,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/728fbcd610244f1fafecd4798429b942 is 50, key is test_row_0/C:col10/1731827771520/Put/seqid=0 2024-11-17T07:16:12,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742429_1605 (size=12151) 2024-11-17T07:16:13,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T07:16:13,149 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/ca65211f010045b1a4125f93e3a1615b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ca65211f010045b1a4125f93e3a1615b 2024-11-17T07:16:13,153 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/B of c9fb4027559a6d665f6131d22dd7458a into ca65211f010045b1a4125f93e3a1615b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:13,153 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:13,153 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/B, priority=13, startTime=1731827772716; duration=0sec 2024-11-17T07:16:13,153 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:13,153 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:B 2024-11-17T07:16:13,203 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/cc2f8967d71347299503a7a3df4310d3 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/cc2f8967d71347299503a7a3df4310d3 2024-11-17T07:16:13,207 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/C of c9fb4027559a6d665f6131d22dd7458a into cc2f8967d71347299503a7a3df4310d3(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:13,207 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:13,207 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/C, priority=13, startTime=1731827772716; duration=0sec 2024-11-17T07:16:13,207 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:13,207 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:C 2024-11-17T07:16:13,246 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/728fbcd610244f1fafecd4798429b942 2024-11-17T07:16:13,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/40ff131254044125b58783399ed76bc6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/40ff131254044125b58783399ed76bc6 2024-11-17T07:16:13,254 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/40ff131254044125b58783399ed76bc6, entries=150, sequenceid=152, filesize=30.4 K 2024-11-17T07:16:13,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/54830c508d144e59940b86ec8e11cf7c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/54830c508d144e59940b86ec8e11cf7c 2024-11-17T07:16:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,258 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/54830c508d144e59940b86ec8e11cf7c, entries=150, sequenceid=152, filesize=11.9 K 2024-11-17T07:16:13,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/728fbcd610244f1fafecd4798429b942 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/728fbcd610244f1fafecd4798429b942 2024-11-17T07:16:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,262 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/728fbcd610244f1fafecd4798429b942, entries=150, sequenceid=152, filesize=11.9 K 2024-11-17T07:16:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,263 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for c9fb4027559a6d665f6131d22dd7458a in 478ms, sequenceid=152, compaction requested=false 2024-11-17T07:16:13,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:13,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:13,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-17T07:16:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-17T07:16:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-17T07:16:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 785 msec 2024-11-17T07:16:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 789 msec 2024-11-17T07:16:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-17T07:16:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,618 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:16:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,618 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:16:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-17T07:16:13,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,621 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:16:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,622 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:16:13,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:16:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T07:16:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T07:16:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:13,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:16:13,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:13,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:13,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:13,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:13,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:13,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:13,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117be7a2488c94b40b69bdbe204e81de460_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827773744/Put/seqid=0 2024-11-17T07:16:13,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742430_1606 (size=22268) 2024-11-17T07:16:13,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T07:16:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:13,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:13,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:13,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827773806, max timestamp=1731827773821Num uniques 859; 2024-11-17T07:16:13,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:16:13,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827773806, max timestamp=1731827773821Num uniques 859; 2024-11-17T07:16:13,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:16:13,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827773806, max timestamp=1731827773821Num uniques 859; 2024-11-17T07:16:13,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:16:13,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:13,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827833905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:13,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:13,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T07:16:13,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:13,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:13,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:13,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:13,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:13,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:13,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T07:16:14,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:14,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827834016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:14,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:14,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:14,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:14,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T07:16:14,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:14,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:14,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:14,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:14,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:14,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:14,159 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:14,162 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117be7a2488c94b40b69bdbe204e81de460_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117be7a2488c94b40b69bdbe204e81de460_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:14,163 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/da0fa79c776a4b0184a6fb4ef805ec3e, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:14,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/da0fa79c776a4b0184a6fb4ef805ec3e is 175, key is test_row_0/A:col10/1731827773744/Put/seqid=0 2024-11-17T07:16:14,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742431_1607 (size=65669) 2024-11-17T07:16:14,170 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=164, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/da0fa79c776a4b0184a6fb4ef805ec3e 2024-11-17T07:16:14,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/ecfd843dfd49456185da3c74c1f3af16 is 50, key is test_row_0/B:col10/1731827773744/Put/seqid=0 2024-11-17T07:16:14,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742432_1608 (size=9757) 2024-11-17T07:16:14,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/ecfd843dfd49456185da3c74c1f3af16 2024-11-17T07:16:14,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/edc60969feee4a08a3fee752ee4a5059 is 50, key is test_row_0/C:col10/1731827773744/Put/seqid=0 2024-11-17T07:16:14,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742433_1609 (size=9757) 2024-11-17T07:16:14,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:14,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827834227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:14,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:14,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T07:16:14,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:14,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:14,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:14,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:14,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:14,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T07:16:14,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T07:16:14,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:14,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:14,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:14,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:14,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:14,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:14,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T07:16:14,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:14,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:14,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:14,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:14,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:14,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:14,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827834537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:14,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/edc60969feee4a08a3fee752ee4a5059 2024-11-17T07:16:14,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/da0fa79c776a4b0184a6fb4ef805ec3e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/da0fa79c776a4b0184a6fb4ef805ec3e 2024-11-17T07:16:14,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/da0fa79c776a4b0184a6fb4ef805ec3e, entries=350, sequenceid=164, filesize=64.1 K 2024-11-17T07:16:14,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/ecfd843dfd49456185da3c74c1f3af16 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ecfd843dfd49456185da3c74c1f3af16 2024-11-17T07:16:14,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ecfd843dfd49456185da3c74c1f3af16, entries=100, sequenceid=164, filesize=9.5 K 2024-11-17T07:16:14,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/edc60969feee4a08a3fee752ee4a5059 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/edc60969feee4a08a3fee752ee4a5059 2024-11-17T07:16:14,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/edc60969feee4a08a3fee752ee4a5059, entries=100, sequenceid=164, filesize=9.5 K 2024-11-17T07:16:14,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c9fb4027559a6d665f6131d22dd7458a in 867ms, sequenceid=164, compaction requested=true 2024-11-17T07:16:14,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:14,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:16:14,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:14,611 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:14,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:16:14,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:14,611 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:14,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:16:14,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:14,612 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128137 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:14,612 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:14,612 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/B is initiating minor compaction (all files) 2024-11-17T07:16:14,612 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/A is initiating minor compaction (all files) 2024-11-17T07:16:14,612 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/B in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:14,612 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/A in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:14,612 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ca65211f010045b1a4125f93e3a1615b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/54830c508d144e59940b86ec8e11cf7c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ecfd843dfd49456185da3c74c1f3af16] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=33.5 K 2024-11-17T07:16:14,612 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/2516f1ddb1824665bc194d6186e97a06, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/40ff131254044125b58783399ed76bc6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/da0fa79c776a4b0184a6fb4ef805ec3e] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=125.1 K 2024-11-17T07:16:14,612 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:14,612 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/2516f1ddb1824665bc194d6186e97a06, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/40ff131254044125b58783399ed76bc6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/da0fa79c776a4b0184a6fb4ef805ec3e] 2024-11-17T07:16:14,612 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting ca65211f010045b1a4125f93e3a1615b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731827770275 2024-11-17T07:16:14,613 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2516f1ddb1824665bc194d6186e97a06, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731827770275 2024-11-17T07:16:14,613 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40ff131254044125b58783399ed76bc6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731827771505 2024-11-17T07:16:14,613 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 54830c508d144e59940b86ec8e11cf7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731827771505 2024-11-17T07:16:14,613 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting da0fa79c776a4b0184a6fb4ef805ec3e, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731827773689 2024-11-17T07:16:14,613 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting ecfd843dfd49456185da3c74c1f3af16, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731827773732 2024-11-17T07:16:14,618 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:14,619 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#B#compaction#520 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:14,619 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/ed5544e388384b4091cd3add9a89d5ab is 50, key is test_row_0/B:col10/1731827773744/Put/seqid=0 2024-11-17T07:16:14,620 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117f1c5bdc4d6354ef19c33ec6da39d3ba1_c9fb4027559a6d665f6131d22dd7458a store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:14,622 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117f1c5bdc4d6354ef19c33ec6da39d3ba1_c9fb4027559a6d665f6131d22dd7458a, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:14,622 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f1c5bdc4d6354ef19c33ec6da39d3ba1_c9fb4027559a6d665f6131d22dd7458a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:14,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742434_1610 (size=12561) 2024-11-17T07:16:14,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742435_1611 (size=4469) 2024-11-17T07:16:14,629 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/ed5544e388384b4091cd3add9a89d5ab as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ed5544e388384b4091cd3add9a89d5ab 2024-11-17T07:16:14,632 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/B of c9fb4027559a6d665f6131d22dd7458a into ed5544e388384b4091cd3add9a89d5ab(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:14,632 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:14,632 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/B, priority=13, startTime=1731827774611; duration=0sec 2024-11-17T07:16:14,632 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:14,632 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:B 2024-11-17T07:16:14,632 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:14,633 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:14,633 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/C is initiating minor compaction (all files) 2024-11-17T07:16:14,633 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/C in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:14,633 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/cc2f8967d71347299503a7a3df4310d3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/728fbcd610244f1fafecd4798429b942, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/edc60969feee4a08a3fee752ee4a5059] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=33.5 K 2024-11-17T07:16:14,634 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting cc2f8967d71347299503a7a3df4310d3, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731827770275 2024-11-17T07:16:14,634 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 728fbcd610244f1fafecd4798429b942, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731827771505 2024-11-17T07:16:14,634 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting edc60969feee4a08a3fee752ee4a5059, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731827773732 2024-11-17T07:16:14,639 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#C#compaction#521 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:14,640 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/9fed5e1479b442468d79969b35a0895f is 50, key is test_row_0/C:col10/1731827773744/Put/seqid=0 2024-11-17T07:16:14,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742436_1612 (size=12561) 2024-11-17T07:16:14,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-17T07:16:14,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:14,692 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-17T07:16:14,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:14,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:16:14,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:14,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:16:14,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:14,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:16:14,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117702116acee6b4e37a00db89b4f184ce0_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827773866/Put/seqid=0 2024-11-17T07:16:14,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742437_1613 (size=12304) 2024-11-17T07:16:14,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T07:16:15,033 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#A#compaction#519 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:15,034 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/0bc675ee5829484ebf8e8e914ed437dd is 175, key is test_row_0/A:col10/1731827773744/Put/seqid=0 2024-11-17T07:16:15,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742438_1614 (size=31622) 2024-11-17T07:16:15,047 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/9fed5e1479b442468d79969b35a0895f as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/9fed5e1479b442468d79969b35a0895f 2024-11-17T07:16:15,051 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/C of c9fb4027559a6d665f6131d22dd7458a into 9fed5e1479b442468d79969b35a0895f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:15,051 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:15,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:15,051 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/C, priority=13, startTime=1731827774611; duration=0sec 2024-11-17T07:16:15,051 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:15,051 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:C 2024-11-17T07:16:15,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827835082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:15,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:15,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:15,108 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117702116acee6b4e37a00db89b4f184ce0_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117702116acee6b4e37a00db89b4f184ce0_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:15,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/84f5a953ed864932828b7e144711ed2e, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:15,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/84f5a953ed864932828b7e144711ed2e is 175, key is test_row_0/A:col10/1731827773866/Put/seqid=0 2024-11-17T07:16:15,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742439_1615 (size=31105) 2024-11-17T07:16:15,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827835197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:15,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:15,200 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:15,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827835407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:15,410 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:15,411 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:15,447 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/0bc675ee5829484ebf8e8e914ed437dd as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/0bc675ee5829484ebf8e8e914ed437dd 2024-11-17T07:16:15,451 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/A of c9fb4027559a6d665f6131d22dd7458a into 0bc675ee5829484ebf8e8e914ed437dd(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:15,451 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:15,451 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/A, priority=13, startTime=1731827774611; duration=0sec 2024-11-17T07:16:15,451 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:15,451 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:A 2024-11-17T07:16:15,513 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=202, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/84f5a953ed864932828b7e144711ed2e 2024-11-17T07:16:15,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/94b7bf969f6d4dd992191606874d929b is 50, key is test_row_0/B:col10/1731827773866/Put/seqid=0 2024-11-17T07:16:15,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742440_1616 (size=12151) 2024-11-17T07:16:15,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:15,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827835717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:15,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T07:16:15,932 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/94b7bf969f6d4dd992191606874d929b 2024-11-17T07:16:15,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/2828e388a1fe4f0d8995a19bc4a4ba95 is 50, key is test_row_0/C:col10/1731827773866/Put/seqid=0 2024-11-17T07:16:15,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742441_1617 (size=12151) 2024-11-17T07:16:16,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827836227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,232 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:16,343 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/2828e388a1fe4f0d8995a19bc4a4ba95 2024-11-17T07:16:16,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/84f5a953ed864932828b7e144711ed2e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/84f5a953ed864932828b7e144711ed2e 2024-11-17T07:16:16,350 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/84f5a953ed864932828b7e144711ed2e, entries=150, sequenceid=202, filesize=30.4 K 2024-11-17T07:16:16,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/94b7bf969f6d4dd992191606874d929b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/94b7bf969f6d4dd992191606874d929b 2024-11-17T07:16:16,354 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/94b7bf969f6d4dd992191606874d929b, entries=150, sequenceid=202, filesize=11.9 K 2024-11-17T07:16:16,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/2828e388a1fe4f0d8995a19bc4a4ba95 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/2828e388a1fe4f0d8995a19bc4a4ba95 2024-11-17T07:16:16,358 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/2828e388a1fe4f0d8995a19bc4a4ba95, entries=150, sequenceid=202, filesize=11.9 K 2024-11-17T07:16:16,359 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c9fb4027559a6d665f6131d22dd7458a in 1668ms, sequenceid=202, compaction requested=false 2024-11-17T07:16:16,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:16,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:16,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-17T07:16:16,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-17T07:16:16,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-17T07:16:16,362 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7380 sec 2024-11-17T07:16:16,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.7440 sec 2024-11-17T07:16:16,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:16,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-17T07:16:16,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:16,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:16,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:16,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:16,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:16,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:16,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111730dc8d0f8b774a7fb0fdc89f4d874d26_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827775070/Put/seqid=0 2024-11-17T07:16:16,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742442_1618 (size=14794) 2024-11-17T07:16:16,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:16,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827836431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:16,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827836431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:16,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827776423, max timestamp=1731827776434Num uniques 859; 2024-11-17T07:16:16,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:16,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:16:16,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827836433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:16,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827776423, max timestamp=1731827776434Num uniques 859; 2024-11-17T07:16:16,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827776423, max timestamp=1731827776434Num uniques 859; 2024-11-17T07:16:16,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:16:16,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:16:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:16,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:16,536 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:16,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827836531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,537 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:16,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827836547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:16,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827836547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:16,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827836548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:16,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827836647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:16,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:16,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:16,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827836757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:16,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827836758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:16,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827836758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:16,801 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:16,804 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111730dc8d0f8b774a7fb0fdc89f4d874d26_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111730dc8d0f8b774a7fb0fdc89f4d874d26_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:16,805 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/a19f764895d44b8da94b7114777fea92, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:16,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/a19f764895d44b8da94b7114777fea92 is 175, key is test_row_0/A:col10/1731827775070/Put/seqid=0 2024-11-17T07:16:16,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742443_1619 (size=39749) 2024-11-17T07:16:16,809 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/a19f764895d44b8da94b7114777fea92 2024-11-17T07:16:16,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/07db7ceedc5e46c19553c39a3fa734ae is 50, key is test_row_0/B:col10/1731827775070/Put/seqid=0 2024-11-17T07:16:16,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742444_1620 (size=12151) 2024-11-17T07:16:16,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:16,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827836857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:16,861 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:17,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827837067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:17,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827837067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:17,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827837067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,072 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:17,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:17,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827837167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:17,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:17,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/07db7ceedc5e46c19553c39a3fa734ae 2024-11-17T07:16:17,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/0eedafa788f24d56a16d4c563d16b26e is 50, key is test_row_0/C:col10/1731827775070/Put/seqid=0 2024-11-17T07:16:17,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742445_1621 (size=12151) 2024-11-17T07:16:17,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:17,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827837237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:17,242 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_0' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:17,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827837577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:17,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827837577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:17,580 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:17,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827837577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:17,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/0eedafa788f24d56a16d4c563d16b26e 2024-11-17T07:16:17,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/a19f764895d44b8da94b7114777fea92 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a19f764895d44b8da94b7114777fea92 2024-11-17T07:16:17,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a19f764895d44b8da94b7114777fea92, entries=200, sequenceid=215, filesize=38.8 K 2024-11-17T07:16:17,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/07db7ceedc5e46c19553c39a3fa734ae as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/07db7ceedc5e46c19553c39a3fa734ae 2024-11-17T07:16:17,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/07db7ceedc5e46c19553c39a3fa734ae, entries=150, sequenceid=215, filesize=11.9 K 2024-11-17T07:16:17,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/0eedafa788f24d56a16d4c563d16b26e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/0eedafa788f24d56a16d4c563d16b26e 2024-11-17T07:16:17,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/0eedafa788f24d56a16d4c563d16b26e, entries=150, sequenceid=215, filesize=11.9 K 2024-11-17T07:16:17,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c9fb4027559a6d665f6131d22dd7458a in 1254ms, sequenceid=215, compaction requested=true 2024-11-17T07:16:17,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:17,645 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:17,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:16:17,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:17,645 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:17,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:16:17,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:17,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:16:17,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:17,646 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102476 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:17,646 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/A is initiating minor compaction (all files) 2024-11-17T07:16:17,646 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/A in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:17,646 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/0bc675ee5829484ebf8e8e914ed437dd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/84f5a953ed864932828b7e144711ed2e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a19f764895d44b8da94b7114777fea92] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=100.1 K 2024-11-17T07:16:17,646 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:17,646 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:17,646 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/B is initiating minor compaction (all files) 2024-11-17T07:16:17,646 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/0bc675ee5829484ebf8e8e914ed437dd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/84f5a953ed864932828b7e144711ed2e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a19f764895d44b8da94b7114777fea92] 2024-11-17T07:16:17,646 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/B in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:17,646 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ed5544e388384b4091cd3add9a89d5ab, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/94b7bf969f6d4dd992191606874d929b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/07db7ceedc5e46c19553c39a3fa734ae] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=36.0 K 2024-11-17T07:16:17,647 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc675ee5829484ebf8e8e914ed437dd, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731827771505 2024-11-17T07:16:17,647 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting ed5544e388384b4091cd3add9a89d5ab, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731827771505 2024-11-17T07:16:17,647 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b7bf969f6d4dd992191606874d929b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731827773866 2024-11-17T07:16:17,647 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84f5a953ed864932828b7e144711ed2e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731827773866 2024-11-17T07:16:17,647 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting a19f764895d44b8da94b7114777fea92, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731827775070 2024-11-17T07:16:17,647 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 07db7ceedc5e46c19553c39a3fa734ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731827775070 2024-11-17T07:16:17,656 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#B#compaction#528 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:17,656 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/c22e051920b04cc59705685376012575 is 50, key is test_row_0/B:col10/1731827775070/Put/seqid=0 2024-11-17T07:16:17,657 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:17,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742446_1622 (size=12663) 2024-11-17T07:16:17,666 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117ff644652313b4a4aa90f39c9861be377_c9fb4027559a6d665f6131d22dd7458a store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:17,668 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117ff644652313b4a4aa90f39c9861be377_c9fb4027559a6d665f6131d22dd7458a, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:17,668 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117ff644652313b4a4aa90f39c9861be377_c9fb4027559a6d665f6131d22dd7458a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:17,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:17,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-17T07:16:17,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:17,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:16:17,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:17,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:16:17,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:17,682 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-17T07:16:17,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742447_1623 (size=4469) 2024-11-17T07:16:17,687 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#A#compaction#529 average throughput is 0.81 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:17,687 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/a6f1cf360b82464ab2aa72b7b8f622c8 is 175, key is test_row_0/A:col10/1731827775070/Put/seqid=0 2024-11-17T07:16:17,690 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/c22e051920b04cc59705685376012575 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/c22e051920b04cc59705685376012575 2024-11-17T07:16:17,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f95c982a18e843a6bca8e3c9d80bd5d4_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827776522/Put/seqid=0 2024-11-17T07:16:17,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742448_1624 (size=31617) 2024-11-17T07:16:17,697 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/B of c9fb4027559a6d665f6131d22dd7458a into c22e051920b04cc59705685376012575(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:17,697 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:17,697 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/B, priority=13, startTime=1731827777645; duration=0sec 2024-11-17T07:16:17,697 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:17,697 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:B 2024-11-17T07:16:17,697 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:17,699 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:17,699 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/C is initiating minor compaction (all files) 2024-11-17T07:16:17,699 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/C in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:17,699 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/9fed5e1479b442468d79969b35a0895f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/2828e388a1fe4f0d8995a19bc4a4ba95, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/0eedafa788f24d56a16d4c563d16b26e] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=36.0 K 2024-11-17T07:16:17,700 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fed5e1479b442468d79969b35a0895f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731827771505 2024-11-17T07:16:17,700 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 2828e388a1fe4f0d8995a19bc4a4ba95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731827773866 2024-11-17T07:16:17,700 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eedafa788f24d56a16d4c563d16b26e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731827775070 2024-11-17T07:16:17,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742449_1625 (size=14794) 2024-11-17T07:16:17,710 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#C#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:17,711 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/417c063f2b6c42469584a9ad4a50ac80 is 50, key is test_row_0/C:col10/1731827775070/Put/seqid=0 2024-11-17T07:16:17,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742450_1626 (size=12663) 2024-11-17T07:16:17,722 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/417c063f2b6c42469584a9ad4a50ac80 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/417c063f2b6c42469584a9ad4a50ac80 2024-11-17T07:16:17,725 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/C of c9fb4027559a6d665f6131d22dd7458a into 417c063f2b6c42469584a9ad4a50ac80(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:17,725 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:17,725 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/C, priority=13, startTime=1731827777645; duration=0sec 2024-11-17T07:16:17,725 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:17,725 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:C 2024-11-17T07:16:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827837721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:17,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-17T07:16:17,787 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:16:17,788 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:16:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-17T07:16:17,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T07:16:17,790 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:16:17,790 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:16:17,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:16:17,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:17,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827837837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:17,840 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T07:16:17,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T07:16:17,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:17,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:17,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:17,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:17,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827838047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:18,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T07:16:18,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:18,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:18,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:18,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,100 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/a6f1cf360b82464ab2aa72b7b8f622c8 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a6f1cf360b82464ab2aa72b7b8f622c8 2024-11-17T07:16:18,104 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/A of c9fb4027559a6d665f6131d22dd7458a into a6f1cf360b82464ab2aa72b7b8f622c8(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:18,104 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:18,104 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/A, priority=13, startTime=1731827777645; duration=0sec 2024-11-17T07:16:18,104 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:18,104 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:A 2024-11-17T07:16:18,106 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:18,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T07:16:18,109 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117f95c982a18e843a6bca8e3c9d80bd5d4_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f95c982a18e843a6bca8e3c9d80bd5d4_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:18,110 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/9b595867af3646cb89cb74cf6d191bff, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:18,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/9b595867af3646cb89cb74cf6d191bff is 175, key is test_row_0/A:col10/1731827776522/Put/seqid=0 2024-11-17T07:16:18,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742451_1627 (size=39749) 2024-11-17T07:16:18,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T07:16:18,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:18,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:18,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:18,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827838357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:18,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T07:16:18,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:18,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:18,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:18,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T07:16:18,514 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/9b595867af3646cb89cb74cf6d191bff 2024-11-17T07:16:18,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/480c48ef4628476ba7cafd1bc93e1dc1 is 50, key is test_row_0/B:col10/1731827776522/Put/seqid=0 2024-11-17T07:16:18,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742452_1628 (size=12151) 2024-11-17T07:16:18,527 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/480c48ef4628476ba7cafd1bc93e1dc1 2024-11-17T07:16:18,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/e131957fae524633898283e68ede87f7 is 50, key is test_row_0/C:col10/1731827776522/Put/seqid=0 2024-11-17T07:16:18,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742453_1629 (size=12151) 2024-11-17T07:16:18,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T07:16:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:18,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827838587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:18,590 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:18,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827838597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:18,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827838597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:18,599 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:18,599 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:18,600 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:18,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T07:16:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:18,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T07:16:18,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:18,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:18,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:18,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:18,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827838867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:18,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:18,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:18,869 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T07:16:18,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/e131957fae524633898283e68ede87f7 2024-11-17T07:16:18,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/9b595867af3646cb89cb74cf6d191bff as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/9b595867af3646cb89cb74cf6d191bff 2024-11-17T07:16:18,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/9b595867af3646cb89cb74cf6d191bff, entries=200, sequenceid=252, filesize=38.8 K 2024-11-17T07:16:18,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/480c48ef4628476ba7cafd1bc93e1dc1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/480c48ef4628476ba7cafd1bc93e1dc1 2024-11-17T07:16:18,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/480c48ef4628476ba7cafd1bc93e1dc1, entries=150, sequenceid=252, filesize=11.9 K 2024-11-17T07:16:18,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/e131957fae524633898283e68ede87f7 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/e131957fae524633898283e68ede87f7 2024-11-17T07:16:18,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/e131957fae524633898283e68ede87f7, entries=150, sequenceid=252, filesize=11.9 K 2024-11-17T07:16:18,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c9fb4027559a6d665f6131d22dd7458a in 1272ms, sequenceid=252, compaction requested=false 2024-11-17T07:16:18,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:19,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-17T07:16:19,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:19,014 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:16:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:19,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:19,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d05f005c59844e029dc57253d2835dd4_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827777693/Put/seqid=0 2024-11-17T07:16:19,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742454_1630 (size=12404) 2024-11-17T07:16:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:19,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:19,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827779315, max timestamp=1731827779334Num uniques 859; 2024-11-17T07:16:19,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827779315, max timestamp=1731827779334Num uniques 859; 2024-11-17T07:16:19,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827779315, max timestamp=1731827779334Num uniques 859; 2024-11-17T07:16:19,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:16:19,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:16:19,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:16:19,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:19,428 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d05f005c59844e029dc57253d2835dd4_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d05f005c59844e029dc57253d2835dd4_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/a609cdf26dd64149a285685262980fd5, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:19,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/a609cdf26dd64149a285685262980fd5 is 175, key is test_row_0/A:col10/1731827777693/Put/seqid=0 2024-11-17T07:16:19,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742455_1631 (size=31205) 2024-11-17T07:16:19,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:19,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827839428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:19,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:19,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:19,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:19,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827839537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:19,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827839747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:19,833 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/a609cdf26dd64149a285685262980fd5 2024-11-17T07:16:19,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/8be0796a0b544736b7667cd6f516db04 is 50, key is test_row_0/B:col10/1731827777693/Put/seqid=0 2024-11-17T07:16:19,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742456_1632 (size=12251) 2024-11-17T07:16:19,842 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/8be0796a0b544736b7667cd6f516db04 2024-11-17T07:16:19,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/1585382ef588421097addc85f0a6b0b5 is 50, key is test_row_0/C:col10/1731827777693/Put/seqid=0 2024-11-17T07:16:19,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742457_1633 (size=12251) 2024-11-17T07:16:19,852 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/1585382ef588421097addc85f0a6b0b5 2024-11-17T07:16:19,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/a609cdf26dd64149a285685262980fd5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a609cdf26dd64149a285685262980fd5 2024-11-17T07:16:19,857 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a609cdf26dd64149a285685262980fd5, entries=150, sequenceid=265, filesize=30.5 K 2024-11-17T07:16:19,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/8be0796a0b544736b7667cd6f516db04 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/8be0796a0b544736b7667cd6f516db04 2024-11-17T07:16:19,861 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/8be0796a0b544736b7667cd6f516db04, entries=150, sequenceid=265, filesize=12.0 K 2024-11-17T07:16:19,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/1585382ef588421097addc85f0a6b0b5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/1585382ef588421097addc85f0a6b0b5 2024-11-17T07:16:19,864 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/1585382ef588421097addc85f0a6b0b5, entries=150, sequenceid=265, filesize=12.0 K 2024-11-17T07:16:19,865 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c9fb4027559a6d665f6131d22dd7458a in 852ms, sequenceid=265, compaction requested=true 2024-11-17T07:16:19,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:19,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:19,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-17T07:16:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-17T07:16:19,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-17T07:16:19,868 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0760 sec 2024-11-17T07:16:19,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.0800 sec 2024-11-17T07:16:19,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:19,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-17T07:16:19,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:19,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:16:19,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:19,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:16:19,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:19,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:16:19,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cea7e9571c124a28bfae042176f1b9b8_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827779417/Put/seqid=0 2024-11-17T07:16:19,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742458_1634 (size=14994) 2024-11-17T07:16:19,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827839914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:19,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-17T07:16:19,937 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:16:19,938 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-17T07:16:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-17T07:16:19,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T07:16:19,939 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-17T07:16:19,940 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-17T07:16:19,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-17T07:16:20,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:20,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827840027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:20,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:20,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T07:16:20,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827840057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:20,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:20,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:20,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:20,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:20,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:20,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827840237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,241 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:20,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:20,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:20,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:20,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:20,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T07:16:20,292 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:20,295 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cea7e9571c124a28bfae042176f1b9b8_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cea7e9571c124a28bfae042176f1b9b8_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:20,296 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/70161a35bea94472ae7b024acbd525f6, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:20,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/70161a35bea94472ae7b024acbd525f6 is 175, key is test_row_0/A:col10/1731827779417/Put/seqid=0 2024-11-17T07:16:20,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742459_1635 (size=39949) 2024-11-17T07:16:20,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:20,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:20,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:20,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:20,399 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:20,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827840547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:20,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:20,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:20,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:20,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:20,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T07:16:20,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827840567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:20,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:20,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44590 deadline: 1731827840617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:20,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44578 deadline: 1731827840617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:20,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:20,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44540 deadline: 1731827840617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:20,621 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:20,621 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_2' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:20,621 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 1dd62e24b303,36419,1731827614816 for 'test_row_1' in c9fb4027559a6d665f6131d22dd7458a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor39.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-17T07:16:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:20,701 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/70161a35bea94472ae7b024acbd525f6 2024-11-17T07:16:20,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:20,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:20,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:20,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:20,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/3246ac1079f44da9a420803c72859c3b is 50, key is test_row_0/B:col10/1731827779417/Put/seqid=0 2024-11-17T07:16:20,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742460_1636 (size=12301) 2024-11-17T07:16:20,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:20,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:20,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:20,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:20,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:21,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44610 deadline: 1731827841057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:21,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:21,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:21,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:21,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T07:16:21,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/3246ac1079f44da9a420803c72859c3b 2024-11-17T07:16:21,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/11c5fc8f2ac4454aac3955086d52949e is 50, key is test_row_0/C:col10/1731827779417/Put/seqid=0 2024-11-17T07:16:21,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742461_1637 (size=12301) 2024-11-17T07:16:21,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:21,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:21,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:21,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:21,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:21,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:21,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/11c5fc8f2ac4454aac3955086d52949e 2024-11-17T07:16:21,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/70161a35bea94472ae7b024acbd525f6 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/70161a35bea94472ae7b024acbd525f6 2024-11-17T07:16:21,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/70161a35bea94472ae7b024acbd525f6, entries=200, sequenceid=303, filesize=39.0 K 2024-11-17T07:16:21,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/3246ac1079f44da9a420803c72859c3b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/3246ac1079f44da9a420803c72859c3b 2024-11-17T07:16:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/3246ac1079f44da9a420803c72859c3b, entries=150, sequenceid=303, filesize=12.0 K 2024-11-17T07:16:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/11c5fc8f2ac4454aac3955086d52949e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/11c5fc8f2ac4454aac3955086d52949e 2024-11-17T07:16:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/11c5fc8f2ac4454aac3955086d52949e, entries=150, sequenceid=303, filesize=12.0 K 2024-11-17T07:16:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for c9fb4027559a6d665f6131d22dd7458a in 1657ms, sequenceid=303, compaction requested=true 2024-11-17T07:16:21,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,539 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:16:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:16:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:21,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:16:21,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:21,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:16:21,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,539 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:16:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,540 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142520 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:16:21,540 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/A is initiating minor compaction (all files) 2024-11-17T07:16:21,540 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/A in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,540 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a6f1cf360b82464ab2aa72b7b8f622c8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/9b595867af3646cb89cb74cf6d191bff, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a609cdf26dd64149a285685262980fd5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/70161a35bea94472ae7b024acbd525f6] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=139.2 K 2024-11-17T07:16:21,540 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,541 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a6f1cf360b82464ab2aa72b7b8f622c8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/9b595867af3646cb89cb74cf6d191bff, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a609cdf26dd64149a285685262980fd5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/70161a35bea94472ae7b024acbd525f6] 2024-11-17T07:16:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,541 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6f1cf360b82464ab2aa72b7b8f622c8, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731827775070 2024-11-17T07:16:21,541 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:16:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,541 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/B is initiating minor compaction (all files) 2024-11-17T07:16:21,541 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/B in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,541 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/c22e051920b04cc59705685376012575, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/480c48ef4628476ba7cafd1bc93e1dc1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/8be0796a0b544736b7667cd6f516db04, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/3246ac1079f44da9a420803c72859c3b] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=48.2 K 2024-11-17T07:16:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,541 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b595867af3646cb89cb74cf6d191bff, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731827776499 2024-11-17T07:16:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,542 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting c22e051920b04cc59705685376012575, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731827775070 2024-11-17T07:16:21,542 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting a609cdf26dd64149a285685262980fd5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731827777693 2024-11-17T07:16:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,542 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70161a35bea94472ae7b024acbd525f6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731827779403 2024-11-17T07:16:21,542 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 480c48ef4628476ba7cafd1bc93e1dc1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731827776499 2024-11-17T07:16:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,543 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 8be0796a0b544736b7667cd6f516db04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731827777693 2024-11-17T07:16:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,543 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 3246ac1079f44da9a420803c72859c3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731827779403 2024-11-17T07:16:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,549 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,552 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#B#compaction#541 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,552 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/e768eea17daf4bbe9dbbd2e9ac8eaa2b is 50, key is test_row_0/B:col10/1731827779417/Put/seqid=0 2024-11-17T07:16:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,554 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241117739aecbd1c584c8abd508fdebd26044d_c9fb4027559a6d665f6131d22dd7458a store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:21,556 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241117739aecbd1c584c8abd508fdebd26044d_c9fb4027559a6d665f6131d22dd7458a, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:21,556 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117739aecbd1c584c8abd508fdebd26044d_c9fb4027559a6d665f6131d22dd7458a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742462_1638 (size=12949) 2024-11-17T07:16:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742463_1639 (size=4469) 2024-11-17T07:16:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,571 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#A#compaction#540 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,572 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/9e04f7d99b1c44d4968bd33104494c5e is 175, key is test_row_0/A:col10/1731827779417/Put/seqid=0 2024-11-17T07:16:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742464_1640 (size=31903) 2024-11-17T07:16:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,579 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/9e04f7d99b1c44d4968bd33104494c5e as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/9e04f7d99b1c44d4968bd33104494c5e 2024-11-17T07:16:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,584 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/A of c9fb4027559a6d665f6131d22dd7458a into 9e04f7d99b1c44d4968bd33104494c5e(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:21,584 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:21,584 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/A, priority=12, startTime=1731827781539; duration=0sec 2024-11-17T07:16:21,584 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:21,584 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:A 2024-11-17T07:16:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,584 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-17T07:16:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,585 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-17T07:16:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,585 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/C is initiating minor compaction (all files) 2024-11-17T07:16:21,585 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/C in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,585 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/417c063f2b6c42469584a9ad4a50ac80, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/e131957fae524633898283e68ede87f7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/1585382ef588421097addc85f0a6b0b5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/11c5fc8f2ac4454aac3955086d52949e] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=48.2 K 2024-11-17T07:16:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,586 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 417c063f2b6c42469584a9ad4a50ac80, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731827775070 2024-11-17T07:16:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,586 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting e131957fae524633898283e68ede87f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731827776499 2024-11-17T07:16:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,587 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1585382ef588421097addc85f0a6b0b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731827777693 2024-11-17T07:16:21,587 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11c5fc8f2ac4454aac3955086d52949e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731827779403 2024-11-17T07:16:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:21,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-17T07:16:21,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:21,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:21,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:21,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:21,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:21,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:21,594 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#C#compaction#542 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:21,595 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/1b69ef1f36c442feb1f63c933d505b15 is 50, key is test_row_0/C:col10/1731827779417/Put/seqid=0 2024-11-17T07:16:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742465_1641 (size=12949) 2024-11-17T07:16:21,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174b7f49100bf44b7da24ede4c85962be3_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_1/A:col10/1731827781592/Put/seqid=0 2024-11-17T07:16:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,604 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/1b69ef1f36c442feb1f63c933d505b15 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/1b69ef1f36c442feb1f63c933d505b15 2024-11-17T07:16:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742466_1642 (size=12454) 2024-11-17T07:16:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,606 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,609 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/C of c9fb4027559a6d665f6131d22dd7458a into 1b69ef1f36c442feb1f63c933d505b15(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:21,609 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411174b7f49100bf44b7da24ede4c85962be3_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174b7f49100bf44b7da24ede4c85962be3_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:21,609 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,609 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/C, priority=12, startTime=1731827781539; duration=0sec 2024-11-17T07:16:21,610 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:21,610 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:C 2024-11-17T07:16:21,610 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/f0713d9d2c66408e8f2ec7d629824b19, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/f0713d9d2c66408e8f2ec7d629824b19 is 175, key is test_row_1/A:col10/1731827781592/Put/seqid=0 2024-11-17T07:16:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742467_1643 (size=31251) 2024-11-17T07:16:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,616 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/f0713d9d2c66408e8f2ec7d629824b19 2024-11-17T07:16:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:21,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/6b287098a00c4beb9cd069e4d3114bcd is 50, key is test_row_1/B:col10/1731827781592/Put/seqid=0 2024-11-17T07:16:21,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:21,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:21,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742468_1644 (size=9857) 2024-11-17T07:16:21,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/6b287098a00c4beb9cd069e4d3114bcd 2024-11-17T07:16:21,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/3fd62e511d4d4866ad9369097ccead06 is 50, key is test_row_1/C:col10/1731827781592/Put/seqid=0 2024-11-17T07:16:21,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742469_1645 (size=9857) 2024-11-17T07:16:21,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827781652, max timestamp=1731827781683Num uniques 859; 2024-11-17T07:16:21,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827781652, max timestamp=1731827781683Num uniques 859; 2024-11-17T07:16:21,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:16:21,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:16:21,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731827781652, max timestamp=1731827781683Num uniques 859; 2024-11-17T07:16:21,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/1dd62e24b303:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-17T07:16:21,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827841762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:21,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:21,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:21,768 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:21,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:21,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:21,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-17T07:16:21,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:44552 deadline: 1731827841877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 , the old value is region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-17T07:16:21,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., hostname=1dd62e24b303,36419,1731827614816, seqNum=5 because the exception is null or not the one we care about 2024-11-17T07:16:21,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:21,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:21,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:21,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:21,962 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/e768eea17daf4bbe9dbbd2e9ac8eaa2b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e768eea17daf4bbe9dbbd2e9ac8eaa2b 2024-11-17T07:16:21,966 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/B of c9fb4027559a6d665f6131d22dd7458a into e768eea17daf4bbe9dbbd2e9ac8eaa2b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:21,966 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:21,966 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/B, priority=12, startTime=1731827781539; duration=0sec 2024-11-17T07:16:21,966 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:21,966 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:B 2024-11-17T07:16:22,002 INFO [Thread-2499 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2499. 2024-11-17T07:16:22,002 DEBUG [Thread-2499 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:16:22,002 DEBUG [Thread-2499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,002 DEBUG [Thread-2499 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,002 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:16:22,003 INFO [Thread-2501 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2501. 2024-11-17T07:16:22,003 DEBUG [Thread-2501 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:16:22,003 DEBUG [Thread-2501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,003 DEBUG [Thread-2501 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,003 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:16:22,005 INFO [Thread-2505 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2505. 2024-11-17T07:16:22,005 DEBUG [Thread-2505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:16:22,005 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,005 DEBUG [Thread-2505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:16:22,005 INFO [Thread-2507 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2507. 2024-11-17T07:16:22,005 DEBUG [Thread-2507 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:16:22,006 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,006 DEBUG [Thread-2507 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,006 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:16:22,007 INFO [Thread-2503 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2503. 2024-11-17T07:16:22,007 DEBUG [Thread-2503 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:16:22,007 DEBUG [Thread-2503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,007 DEBUG [Thread-2503 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,007 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:16:22,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/3fd62e511d4d4866ad9369097ccead06 2024-11-17T07:16:22,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/f0713d9d2c66408e8f2ec7d629824b19 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/f0713d9d2c66408e8f2ec7d629824b19 2024-11-17T07:16:22,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/f0713d9d2c66408e8f2ec7d629824b19, entries=150, sequenceid=314, filesize=30.5 K 2024-11-17T07:16:22,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/6b287098a00c4beb9cd069e4d3114bcd as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/6b287098a00c4beb9cd069e4d3114bcd 2024-11-17T07:16:22,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/6b287098a00c4beb9cd069e4d3114bcd, entries=100, sequenceid=314, filesize=9.6 K 2024-11-17T07:16:22,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/3fd62e511d4d4866ad9369097ccead06 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/3fd62e511d4d4866ad9369097ccead06 2024-11-17T07:16:22,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/3fd62e511d4d4866ad9369097ccead06, entries=100, sequenceid=314, filesize=9.6 K 2024-11-17T07:16:22,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c9fb4027559a6d665f6131d22dd7458a in 460ms, sequenceid=314, compaction requested=false 2024-11-17T07:16:22,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:22,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36419 {}] regionserver.HRegion(8855): Flush requested on c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:22,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-17T07:16:22,068 INFO [Thread-2492 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2492. 2024-11-17T07:16:22,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:22,068 DEBUG [Thread-2492 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:16:22,068 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,068 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:16:22,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:22,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:16:22,069 DEBUG [Thread-2492 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:16:22,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:22,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-17T07:16:22,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d725b5dab9864e5bbac67aa40e96a9c6_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_0/A:col10/1731827781760/Put/seqid=0 2024-11-17T07:16:22,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742470_1646 (size=12454) 2024-11-17T07:16:22,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T07:16:22,088 INFO [Thread-2496 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2496. 2024-11-17T07:16:22,088 DEBUG [Thread-2496 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:16:22,088 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,088 DEBUG [Thread-2496 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:22,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:16:22,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:22,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:22,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:22,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:22,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,477 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:22,479 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117d725b5dab9864e5bbac67aa40e96a9c6_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d725b5dab9864e5bbac67aa40e96a9c6_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:22,480 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/86d4365249c14dcaa83fedfb5ac4d7f0, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:22,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/86d4365249c14dcaa83fedfb5ac4d7f0 is 175, key is test_row_0/A:col10/1731827781760/Put/seqid=0 2024-11-17T07:16:22,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742471_1647 (size=31255) 2024-11-17T07:16:22,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:22,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:22,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:22,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:22,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:22,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:22,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,884 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=354, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/86d4365249c14dcaa83fedfb5ac4d7f0 2024-11-17T07:16:22,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/dd64d7e964044033b615d1b37a5899b0 is 50, key is test_row_0/B:col10/1731827781760/Put/seqid=0 2024-11-17T07:16:22,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742472_1648 (size=12301) 2024-11-17T07:16:22,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:22,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:23,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:23,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:23,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:23,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:23,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:23,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:23,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/dd64d7e964044033b615d1b37a5899b0 2024-11-17T07:16:23,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/8508f9fcad5249b996a45b12fe158bc5 is 50, key is test_row_0/C:col10/1731827781760/Put/seqid=0 2024-11-17T07:16:23,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:23,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:23,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:23,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:23,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:23,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:23,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742473_1649 (size=12301) 2024-11-17T07:16:23,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:23,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:23,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:23,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:23,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:23,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:23,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:23,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:23,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. as already flushing 2024-11-17T07:16:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:23,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:23,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T07:16:23,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=354 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/8508f9fcad5249b996a45b12fe158bc5 2024-11-17T07:16:23,704 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/86d4365249c14dcaa83fedfb5ac4d7f0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/86d4365249c14dcaa83fedfb5ac4d7f0 2024-11-17T07:16:23,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/86d4365249c14dcaa83fedfb5ac4d7f0, entries=150, sequenceid=354, filesize=30.5 K 2024-11-17T07:16:23,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/dd64d7e964044033b615d1b37a5899b0 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/dd64d7e964044033b615d1b37a5899b0 2024-11-17T07:16:23,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/dd64d7e964044033b615d1b37a5899b0, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T07:16:23,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/8508f9fcad5249b996a45b12fe158bc5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/8508f9fcad5249b996a45b12fe158bc5 2024-11-17T07:16:23,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/8508f9fcad5249b996a45b12fe158bc5, entries=150, sequenceid=354, filesize=12.0 K 2024-11-17T07:16:23,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=6.71 KB/6870 for c9fb4027559a6d665f6131d22dd7458a in 1645ms, sequenceid=354, compaction requested=true 2024-11-17T07:16:23,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:23,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:A, priority=-2147483648, current under compaction store size is 1 2024-11-17T07:16:23,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:23,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:B, priority=-2147483648, current under compaction store size is 2 2024-11-17T07:16:23,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:23,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c9fb4027559a6d665f6131d22dd7458a:C, priority=-2147483648, current under compaction store size is 3 2024-11-17T07:16:23,713 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:23,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:23,713 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:23,714 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94409 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:23,714 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:23,714 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/A is initiating minor compaction (all files) 2024-11-17T07:16:23,714 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/A in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:23,714 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/B is initiating minor compaction (all files) 2024-11-17T07:16:23,714 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/9e04f7d99b1c44d4968bd33104494c5e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/f0713d9d2c66408e8f2ec7d629824b19, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/86d4365249c14dcaa83fedfb5ac4d7f0] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=92.2 K 2024-11-17T07:16:23,714 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:23,714 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/B in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:23,714 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. files: [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/9e04f7d99b1c44d4968bd33104494c5e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/f0713d9d2c66408e8f2ec7d629824b19, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/86d4365249c14dcaa83fedfb5ac4d7f0] 2024-11-17T07:16:23,714 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e768eea17daf4bbe9dbbd2e9ac8eaa2b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/6b287098a00c4beb9cd069e4d3114bcd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/dd64d7e964044033b615d1b37a5899b0] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=34.3 K 2024-11-17T07:16:23,714 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e04f7d99b1c44d4968bd33104494c5e, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731827779403 2024-11-17T07:16:23,715 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting e768eea17daf4bbe9dbbd2e9ac8eaa2b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731827779403 2024-11-17T07:16:23,715 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0713d9d2c66408e8f2ec7d629824b19, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731827779900 2024-11-17T07:16:23,715 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b287098a00c4beb9cd069e4d3114bcd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731827779900 2024-11-17T07:16:23,715 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86d4365249c14dcaa83fedfb5ac4d7f0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731827781754 2024-11-17T07:16:23,715 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting dd64d7e964044033b615d1b37a5899b0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731827781754 2024-11-17T07:16:23,719 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:23,720 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#B#compaction#549 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:23,721 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/f87aca7811f24882a552a9cf2690f155 is 50, key is test_row_0/B:col10/1731827781760/Put/seqid=0 2024-11-17T07:16:23,721 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411170877887454bb474baf08cfc361043b4d_c9fb4027559a6d665f6131d22dd7458a store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:23,724 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411170877887454bb474baf08cfc361043b4d_c9fb4027559a6d665f6131d22dd7458a, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:23,724 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411170877887454bb474baf08cfc361043b4d_c9fb4027559a6d665f6131d22dd7458a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:23,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742474_1650 (size=13051) 2024-11-17T07:16:23,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742475_1651 (size=4469) 2024-11-17T07:16:23,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36419 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-17T07:16:23,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:23,758 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=6.71 KB heapSize=20.67 KB 2024-11-17T07:16:23,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:23,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:23,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:23,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:23,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:23,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:23,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cea7838862d04ed49de9cd9234a2e4cc_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_1/A:col10/1731827782087/Put/seqid=0 2024-11-17T07:16:23,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742476_1652 (size=7374) 2024-11-17T07:16:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T07:16:24,128 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#A#compaction#550 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:24,128 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/c27baf56a4194c66a60c3b2688375de5 is 175, key is test_row_0/A:col10/1731827781760/Put/seqid=0 2024-11-17T07:16:24,129 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/f87aca7811f24882a552a9cf2690f155 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/f87aca7811f24882a552a9cf2690f155 2024-11-17T07:16:24,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742477_1653 (size=32005) 2024-11-17T07:16:24,133 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/B of c9fb4027559a6d665f6131d22dd7458a into f87aca7811f24882a552a9cf2690f155(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:24,133 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:24,133 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/B, priority=13, startTime=1731827783713; duration=0sec 2024-11-17T07:16:24,133 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-17T07:16:24,133 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:B 2024-11-17T07:16:24,133 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-17T07:16:24,134 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-17T07:16:24,134 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1541): c9fb4027559a6d665f6131d22dd7458a/C is initiating minor compaction (all files) 2024-11-17T07:16:24,134 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c9fb4027559a6d665f6131d22dd7458a/C in TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:24,134 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/1b69ef1f36c442feb1f63c933d505b15, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/3fd62e511d4d4866ad9369097ccead06, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/8508f9fcad5249b996a45b12fe158bc5] into tmpdir=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp, totalSize=34.3 K 2024-11-17T07:16:24,134 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b69ef1f36c442feb1f63c933d505b15, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731827779403 2024-11-17T07:16:24,134 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fd62e511d4d4866ad9369097ccead06, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731827779900 2024-11-17T07:16:24,135 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] compactions.Compactor(225): Compacting 8508f9fcad5249b996a45b12fe158bc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=354, earliestPutTs=1731827781754 2024-11-17T07:16:24,139 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c9fb4027559a6d665f6131d22dd7458a#C#compaction#552 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-17T07:16:24,140 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/c1bb939d9be54767a551c1f804ac065b is 50, key is test_row_0/C:col10/1731827781760/Put/seqid=0 2024-11-17T07:16:24,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742478_1654 (size=13051) 2024-11-17T07:16:24,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:24,168 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117cea7838862d04ed49de9cd9234a2e4cc_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cea7838862d04ed49de9cd9234a2e4cc_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:24,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/35dabe9548804a6d9ff5fe5b584f37e4, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:24,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/35dabe9548804a6d9ff5fe5b584f37e4 is 175, key is test_row_1/A:col10/1731827782087/Put/seqid=0 2024-11-17T07:16:24,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742479_1655 (size=13865) 2024-11-17T07:16:24,535 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/c27baf56a4194c66a60c3b2688375de5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/c27baf56a4194c66a60c3b2688375de5 2024-11-17T07:16:24,538 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/A of c9fb4027559a6d665f6131d22dd7458a into c27baf56a4194c66a60c3b2688375de5(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:24,538 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:24,538 INFO [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/A, priority=13, startTime=1731827783713; duration=0sec 2024-11-17T07:16:24,538 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:24,538 DEBUG [RS:0;1dd62e24b303:36419-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:A 2024-11-17T07:16:24,545 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/c1bb939d9be54767a551c1f804ac065b as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/c1bb939d9be54767a551c1f804ac065b 2024-11-17T07:16:24,548 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c9fb4027559a6d665f6131d22dd7458a/C of c9fb4027559a6d665f6131d22dd7458a into c1bb939d9be54767a551c1f804ac065b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-17T07:16:24,548 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:24,548 INFO [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., storeName=c9fb4027559a6d665f6131d22dd7458a/C, priority=13, startTime=1731827783713; duration=0sec 2024-11-17T07:16:24,548 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-17T07:16:24,548 DEBUG [RS:0;1dd62e24b303:36419-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c9fb4027559a6d665f6131d22dd7458a:C 2024-11-17T07:16:24,572 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=2.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/35dabe9548804a6d9ff5fe5b584f37e4 2024-11-17T07:16:24,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/e98db248f0fc4836a396a9a2f0f3b8f1 is 50, key is test_row_1/B:col10/1731827782087/Put/seqid=0 2024-11-17T07:16:24,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742480_1656 (size=7415) 2024-11-17T07:16:24,638 INFO [Thread-2494 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2494. 2024-11-17T07:16:24,638 DEBUG [Thread-2494 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:16:24,638 INFO [Thread-2490 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2490. 2024-11-17T07:16:24,638 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:24,638 DEBUG [Thread-2490 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:16:24,638 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:24,638 DEBUG [Thread-2494 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:24,639 DEBUG [Thread-2490 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:24,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:16:24,639 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:16:24,648 INFO [Thread-2488 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2488. 2024-11-17T07:16:24,648 DEBUG [Thread-2488 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-17T07:16:24,648 DEBUG [Thread-2488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:24,649 DEBUG [Thread-2488 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:24,649 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:16:24,979 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/e98db248f0fc4836a396a9a2f0f3b8f1 2024-11-17T07:16:24,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/7f83601b98e54cc9a10fc36264d8e5db is 50, key is test_row_1/C:col10/1731827782087/Put/seqid=0 2024-11-17T07:16:24,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742481_1657 (size=7415) 2024-11-17T07:16:25,388 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/7f83601b98e54cc9a10fc36264d8e5db 2024-11-17T07:16:25,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/35dabe9548804a6d9ff5fe5b584f37e4 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/35dabe9548804a6d9ff5fe5b584f37e4 2024-11-17T07:16:25,394 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/35dabe9548804a6d9ff5fe5b584f37e4, entries=50, sequenceid=358, filesize=13.5 K 2024-11-17T07:16:25,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/e98db248f0fc4836a396a9a2f0f3b8f1 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e98db248f0fc4836a396a9a2f0f3b8f1 2024-11-17T07:16:25,397 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e98db248f0fc4836a396a9a2f0f3b8f1, entries=50, sequenceid=358, filesize=7.2 K 2024-11-17T07:16:25,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/7f83601b98e54cc9a10fc36264d8e5db as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/7f83601b98e54cc9a10fc36264d8e5db 2024-11-17T07:16:25,399 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/7f83601b98e54cc9a10fc36264d8e5db, entries=50, sequenceid=358, filesize=7.2 K 2024-11-17T07:16:25,400 INFO [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~20.63 KB/21120, currentSize=20.13 KB/20610 for c9fb4027559a6d665f6131d22dd7458a in 1642ms, sequenceid=358, compaction requested=false 2024-11-17T07:16:25,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for c9fb4027559a6d665f6131d22dd7458a: 2024-11-17T07:16:25,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:25,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/1dd62e24b303:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-17T07:16:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-17T07:16:25,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-17T07:16:25,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.4610 sec 2024-11-17T07:16:25,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 5.4650 sec 2024-11-17T07:16:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-17T07:16:28,118 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:16:28,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-17T07:16:28,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-17T07:16:28,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-17T07:16:28,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-17T07:16:28,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-17T07:16:28,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 153 2024-11-17T07:16:28,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-17T07:16:28,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3236 2024-11-17T07:16:28,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3032 2024-11-17T07:16:28,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2969 2024-11-17T07:16:28,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3235 2024-11-17T07:16:28,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3026 2024-11-17T07:16:28,118 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-17T07:16:28,118 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T07:16:28,118 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T07:16:28,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:28,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:28,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:16:28,119 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-17T07:16:28,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-17T07:16:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T07:16:28,121 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827788121"}]},"ts":"1731827788121"} 2024-11-17T07:16:28,123 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-17T07:16:28,123 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-17T07:16:28,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-17T07:16:28,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, UNASSIGN}] 2024-11-17T07:16:28,125 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, UNASSIGN 2024-11-17T07:16:28,125 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=c9fb4027559a6d665f6131d22dd7458a, regionState=CLOSING, regionLocation=1dd62e24b303,36419,1731827614816 2024-11-17T07:16:28,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, UNASSIGN because future has completed 2024-11-17T07:16:28,126 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-17T07:16:28,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816}] 2024-11-17T07:16:28,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T07:16:28,278 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:28,278 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-17T07:16:28,278 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing c9fb4027559a6d665f6131d22dd7458a, disabling compactions & flushes 2024-11-17T07:16:28,278 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:28,278 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:28,278 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. after waiting 0 ms 2024-11-17T07:16:28,278 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:28,279 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing c9fb4027559a6d665f6131d22dd7458a 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-17T07:16:28,279 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=A 2024-11-17T07:16:28,279 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:28,279 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=B 2024-11-17T07:16:28,279 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:28,279 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c9fb4027559a6d665f6131d22dd7458a, store=C 2024-11-17T07:16:28,279 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-17T07:16:28,283 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e7d79f93d4f64981b957e3258b674780_c9fb4027559a6d665f6131d22dd7458a is 50, key is test_row_1/A:col10/1731827784637/Put/seqid=0 2024-11-17T07:16:28,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742482_1658 (size=9914) 2024-11-17T07:16:28,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T07:16:28,687 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T07:16:28,690 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241117e7d79f93d4f64981b957e3258b674780_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e7d79f93d4f64981b957e3258b674780_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:28,691 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/76fc7ecdbb2a4855a53a7e0e18f8bf4d, store: [table=TestAcidGuarantees family=A region=c9fb4027559a6d665f6131d22dd7458a] 2024-11-17T07:16:28,691 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/76fc7ecdbb2a4855a53a7e0e18f8bf4d is 175, key is test_row_1/A:col10/1731827784637/Put/seqid=0 2024-11-17T07:16:28,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742483_1659 (size=22561) 2024-11-17T07:16:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T07:16:29,094 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/76fc7ecdbb2a4855a53a7e0e18f8bf4d 2024-11-17T07:16:29,099 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/064171b6ef0d44a1a255e4d6c6963ec5 is 50, key is test_row_1/B:col10/1731827784637/Put/seqid=0 2024-11-17T07:16:29,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742484_1660 (size=9857) 2024-11-17T07:16:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T07:16:29,503 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/064171b6ef0d44a1a255e4d6c6963ec5 2024-11-17T07:16:29,508 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/a5afd0a54d4e4cf390a639b7f681e5fd is 50, key is test_row_1/C:col10/1731827784637/Put/seqid=0 2024-11-17T07:16:29,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742485_1661 (size=9857) 2024-11-17T07:16:29,911 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/a5afd0a54d4e4cf390a639b7f681e5fd 2024-11-17T07:16:29,915 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/A/76fc7ecdbb2a4855a53a7e0e18f8bf4d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/76fc7ecdbb2a4855a53a7e0e18f8bf4d 2024-11-17T07:16:29,918 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/76fc7ecdbb2a4855a53a7e0e18f8bf4d, entries=100, sequenceid=367, filesize=22.0 K 2024-11-17T07:16:29,919 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/B/064171b6ef0d44a1a255e4d6c6963ec5 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/064171b6ef0d44a1a255e4d6c6963ec5 2024-11-17T07:16:29,922 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/064171b6ef0d44a1a255e4d6c6963ec5, entries=100, sequenceid=367, filesize=9.6 K 2024-11-17T07:16:29,923 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/.tmp/C/a5afd0a54d4e4cf390a639b7f681e5fd as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/a5afd0a54d4e4cf390a639b7f681e5fd 2024-11-17T07:16:29,928 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/a5afd0a54d4e4cf390a639b7f681e5fd, entries=100, sequenceid=367, filesize=9.6 K 2024-11-17T07:16:29,929 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c9fb4027559a6d665f6131d22dd7458a in 1651ms, sequenceid=367, compaction requested=true 2024-11-17T07:16:29,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/fd10515686c84d4588235008ac79f59c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/b357a066e8814b758f3824d57cdb168c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/493905d53e9a4b74a10182a7ea756f61, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/374522f116f047a4a6a4b833921c961d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/23b2256d362642f2b5c8da2503586f5e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/63090727187a461eaff7076db6c08b5c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/6a3e3b159a444f22ba67cde20d704aa5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/2c6094000194435eaf455f29266c8b4e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/39dcaccc8c5b4241b6848cb4b1451c47, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/2516f1ddb1824665bc194d6186e97a06, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/40ff131254044125b58783399ed76bc6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/da0fa79c776a4b0184a6fb4ef805ec3e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/0bc675ee5829484ebf8e8e914ed437dd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/84f5a953ed864932828b7e144711ed2e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a19f764895d44b8da94b7114777fea92, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a6f1cf360b82464ab2aa72b7b8f622c8, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/9b595867af3646cb89cb74cf6d191bff, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a609cdf26dd64149a285685262980fd5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/70161a35bea94472ae7b024acbd525f6, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/9e04f7d99b1c44d4968bd33104494c5e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/f0713d9d2c66408e8f2ec7d629824b19, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/86d4365249c14dcaa83fedfb5ac4d7f0] to archive 2024-11-17T07:16:29,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:16:29,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/fd10515686c84d4588235008ac79f59c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/fd10515686c84d4588235008ac79f59c 2024-11-17T07:16:29,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/b357a066e8814b758f3824d57cdb168c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/b357a066e8814b758f3824d57cdb168c 2024-11-17T07:16:29,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/493905d53e9a4b74a10182a7ea756f61 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/493905d53e9a4b74a10182a7ea756f61 2024-11-17T07:16:29,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/374522f116f047a4a6a4b833921c961d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/374522f116f047a4a6a4b833921c961d 2024-11-17T07:16:29,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/23b2256d362642f2b5c8da2503586f5e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/23b2256d362642f2b5c8da2503586f5e 2024-11-17T07:16:29,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/63090727187a461eaff7076db6c08b5c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/63090727187a461eaff7076db6c08b5c 2024-11-17T07:16:29,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/6a3e3b159a444f22ba67cde20d704aa5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/6a3e3b159a444f22ba67cde20d704aa5 2024-11-17T07:16:29,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/2c6094000194435eaf455f29266c8b4e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/2c6094000194435eaf455f29266c8b4e 2024-11-17T07:16:29,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/39dcaccc8c5b4241b6848cb4b1451c47 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/39dcaccc8c5b4241b6848cb4b1451c47 2024-11-17T07:16:29,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/2516f1ddb1824665bc194d6186e97a06 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/2516f1ddb1824665bc194d6186e97a06 2024-11-17T07:16:29,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/40ff131254044125b58783399ed76bc6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/40ff131254044125b58783399ed76bc6 2024-11-17T07:16:29,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/da0fa79c776a4b0184a6fb4ef805ec3e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/da0fa79c776a4b0184a6fb4ef805ec3e 2024-11-17T07:16:29,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/0bc675ee5829484ebf8e8e914ed437dd to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/0bc675ee5829484ebf8e8e914ed437dd 2024-11-17T07:16:29,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/84f5a953ed864932828b7e144711ed2e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/84f5a953ed864932828b7e144711ed2e 2024-11-17T07:16:29,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a19f764895d44b8da94b7114777fea92 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a19f764895d44b8da94b7114777fea92 2024-11-17T07:16:29,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a6f1cf360b82464ab2aa72b7b8f622c8 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a6f1cf360b82464ab2aa72b7b8f622c8 2024-11-17T07:16:29,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/9b595867af3646cb89cb74cf6d191bff to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/9b595867af3646cb89cb74cf6d191bff 2024-11-17T07:16:29,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a609cdf26dd64149a285685262980fd5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/a609cdf26dd64149a285685262980fd5 2024-11-17T07:16:29,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/70161a35bea94472ae7b024acbd525f6 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/70161a35bea94472ae7b024acbd525f6 2024-11-17T07:16:29,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/9e04f7d99b1c44d4968bd33104494c5e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/9e04f7d99b1c44d4968bd33104494c5e 2024-11-17T07:16:29,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/f0713d9d2c66408e8f2ec7d629824b19 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/f0713d9d2c66408e8f2ec7d629824b19 2024-11-17T07:16:29,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/86d4365249c14dcaa83fedfb5ac4d7f0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/86d4365249c14dcaa83fedfb5ac4d7f0 2024-11-17T07:16:29,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/89e067db16cb47b185e529a3cb98d339, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/85f5bcdb4fda4ca1a6871b45b1969152, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/76bc91ca13614d76bc4712eae4f37882, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/8a91d9fb52b64d4787002ec955dc8fb7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/b612dba8b6554817b59bf6b4a61bbc4d, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/3b87f1432c304d4384448a4f75e09743, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/0fac1e31df4441b8966d7f15ea5006cc, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/29e9bcbebd9849a3ab6f6997202d1555, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ca65211f010045b1a4125f93e3a1615b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e6b2305539344f9a8a93b557235ed89a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/54830c508d144e59940b86ec8e11cf7c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ed5544e388384b4091cd3add9a89d5ab, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ecfd843dfd49456185da3c74c1f3af16, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/94b7bf969f6d4dd992191606874d929b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/c22e051920b04cc59705685376012575, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/07db7ceedc5e46c19553c39a3fa734ae, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/480c48ef4628476ba7cafd1bc93e1dc1, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/8be0796a0b544736b7667cd6f516db04, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e768eea17daf4bbe9dbbd2e9ac8eaa2b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/3246ac1079f44da9a420803c72859c3b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/6b287098a00c4beb9cd069e4d3114bcd, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/dd64d7e964044033b615d1b37a5899b0] to archive 2024-11-17T07:16:29,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:16:29,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/89e067db16cb47b185e529a3cb98d339 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/89e067db16cb47b185e529a3cb98d339 2024-11-17T07:16:29,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/85f5bcdb4fda4ca1a6871b45b1969152 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/85f5bcdb4fda4ca1a6871b45b1969152 2024-11-17T07:16:29,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/76bc91ca13614d76bc4712eae4f37882 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/76bc91ca13614d76bc4712eae4f37882 2024-11-17T07:16:29,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/8a91d9fb52b64d4787002ec955dc8fb7 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/8a91d9fb52b64d4787002ec955dc8fb7 2024-11-17T07:16:29,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/b612dba8b6554817b59bf6b4a61bbc4d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/b612dba8b6554817b59bf6b4a61bbc4d 2024-11-17T07:16:29,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/3b87f1432c304d4384448a4f75e09743 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/3b87f1432c304d4384448a4f75e09743 2024-11-17T07:16:29,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/0fac1e31df4441b8966d7f15ea5006cc to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/0fac1e31df4441b8966d7f15ea5006cc 2024-11-17T07:16:29,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/29e9bcbebd9849a3ab6f6997202d1555 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/29e9bcbebd9849a3ab6f6997202d1555 2024-11-17T07:16:29,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ca65211f010045b1a4125f93e3a1615b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ca65211f010045b1a4125f93e3a1615b 2024-11-17T07:16:29,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e6b2305539344f9a8a93b557235ed89a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e6b2305539344f9a8a93b557235ed89a 2024-11-17T07:16:29,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/54830c508d144e59940b86ec8e11cf7c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/54830c508d144e59940b86ec8e11cf7c 2024-11-17T07:16:29,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ed5544e388384b4091cd3add9a89d5ab to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ed5544e388384b4091cd3add9a89d5ab 2024-11-17T07:16:29,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ecfd843dfd49456185da3c74c1f3af16 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/ecfd843dfd49456185da3c74c1f3af16 2024-11-17T07:16:29,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/94b7bf969f6d4dd992191606874d929b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/94b7bf969f6d4dd992191606874d929b 2024-11-17T07:16:29,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/c22e051920b04cc59705685376012575 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/c22e051920b04cc59705685376012575 2024-11-17T07:16:29,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/07db7ceedc5e46c19553c39a3fa734ae to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/07db7ceedc5e46c19553c39a3fa734ae 2024-11-17T07:16:29,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/480c48ef4628476ba7cafd1bc93e1dc1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/480c48ef4628476ba7cafd1bc93e1dc1 2024-11-17T07:16:29,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/8be0796a0b544736b7667cd6f516db04 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/8be0796a0b544736b7667cd6f516db04 2024-11-17T07:16:29,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e768eea17daf4bbe9dbbd2e9ac8eaa2b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e768eea17daf4bbe9dbbd2e9ac8eaa2b 2024-11-17T07:16:29,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/3246ac1079f44da9a420803c72859c3b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/3246ac1079f44da9a420803c72859c3b 2024-11-17T07:16:29,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/6b287098a00c4beb9cd069e4d3114bcd to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/6b287098a00c4beb9cd069e4d3114bcd 2024-11-17T07:16:29,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/dd64d7e964044033b615d1b37a5899b0 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/dd64d7e964044033b615d1b37a5899b0 2024-11-17T07:16:29,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/a8f53e68bd624d83889f1218c526e8cf, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/409fe3bd7c9344a69e4c90ed86ae907c, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/944eedc463764249a15aa438b5c17af7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/33f191f075ef4ad7aa89c2a8dfb6215b, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/461448d2b3ea40989e144dc07a8d36c2, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/66a5d000f528424e82932daafc4c4e15, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/bfb92f13c7494ecea9cea68570ca656a, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/bd7421628b3c47a29897e6a3048c5c09, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/cc2f8967d71347299503a7a3df4310d3, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/f5376762a41d452d97505f704953bcf4, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/728fbcd610244f1fafecd4798429b942, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/9fed5e1479b442468d79969b35a0895f, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/edc60969feee4a08a3fee752ee4a5059, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/2828e388a1fe4f0d8995a19bc4a4ba95, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/417c063f2b6c42469584a9ad4a50ac80, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/0eedafa788f24d56a16d4c563d16b26e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/e131957fae524633898283e68ede87f7, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/1585382ef588421097addc85f0a6b0b5, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/1b69ef1f36c442feb1f63c933d505b15, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/11c5fc8f2ac4454aac3955086d52949e, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/3fd62e511d4d4866ad9369097ccead06, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/8508f9fcad5249b996a45b12fe158bc5] to archive 2024-11-17T07:16:29,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-17T07:16:29,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/a8f53e68bd624d83889f1218c526e8cf to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/a8f53e68bd624d83889f1218c526e8cf 2024-11-17T07:16:29,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/409fe3bd7c9344a69e4c90ed86ae907c to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/409fe3bd7c9344a69e4c90ed86ae907c 2024-11-17T07:16:29,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/944eedc463764249a15aa438b5c17af7 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/944eedc463764249a15aa438b5c17af7 2024-11-17T07:16:29,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/33f191f075ef4ad7aa89c2a8dfb6215b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/33f191f075ef4ad7aa89c2a8dfb6215b 2024-11-17T07:16:29,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/461448d2b3ea40989e144dc07a8d36c2 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/461448d2b3ea40989e144dc07a8d36c2 2024-11-17T07:16:29,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/66a5d000f528424e82932daafc4c4e15 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/66a5d000f528424e82932daafc4c4e15 2024-11-17T07:16:29,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/bfb92f13c7494ecea9cea68570ca656a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/bfb92f13c7494ecea9cea68570ca656a 2024-11-17T07:16:29,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/bd7421628b3c47a29897e6a3048c5c09 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/bd7421628b3c47a29897e6a3048c5c09 2024-11-17T07:16:29,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/cc2f8967d71347299503a7a3df4310d3 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/cc2f8967d71347299503a7a3df4310d3 2024-11-17T07:16:29,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/f5376762a41d452d97505f704953bcf4 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/f5376762a41d452d97505f704953bcf4 2024-11-17T07:16:29,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/728fbcd610244f1fafecd4798429b942 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/728fbcd610244f1fafecd4798429b942 2024-11-17T07:16:29,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/9fed5e1479b442468d79969b35a0895f to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/9fed5e1479b442468d79969b35a0895f 2024-11-17T07:16:29,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/edc60969feee4a08a3fee752ee4a5059 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/edc60969feee4a08a3fee752ee4a5059 2024-11-17T07:16:29,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/2828e388a1fe4f0d8995a19bc4a4ba95 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/2828e388a1fe4f0d8995a19bc4a4ba95 2024-11-17T07:16:29,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/417c063f2b6c42469584a9ad4a50ac80 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/417c063f2b6c42469584a9ad4a50ac80 2024-11-17T07:16:29,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/0eedafa788f24d56a16d4c563d16b26e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/0eedafa788f24d56a16d4c563d16b26e 2024-11-17T07:16:29,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/e131957fae524633898283e68ede87f7 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/e131957fae524633898283e68ede87f7 2024-11-17T07:16:29,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/1585382ef588421097addc85f0a6b0b5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/1585382ef588421097addc85f0a6b0b5 2024-11-17T07:16:29,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/1b69ef1f36c442feb1f63c933d505b15 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/1b69ef1f36c442feb1f63c933d505b15 2024-11-17T07:16:29,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/11c5fc8f2ac4454aac3955086d52949e to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/11c5fc8f2ac4454aac3955086d52949e 2024-11-17T07:16:30,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/3fd62e511d4d4866ad9369097ccead06 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/3fd62e511d4d4866ad9369097ccead06 2024-11-17T07:16:30,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/8508f9fcad5249b996a45b12fe158bc5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/8508f9fcad5249b996a45b12fe158bc5 2024-11-17T07:16:30,006 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/recovered.edits/370.seqid, newMaxSeqId=370, maxSeqId=4 2024-11-17T07:16:30,006 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. 2024-11-17T07:16:30,007 DEBUG [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for c9fb4027559a6d665f6131d22dd7458a: Waiting for close lock at 1731827788278Running coprocessor pre-close hooks at 1731827788278Disabling compacts and flushes for region at 1731827788278Disabling writes for close at 1731827788278Obtaining lock to block concurrent updates at 1731827788279 (+1 ms)Preparing flush snapshotting stores in c9fb4027559a6d665f6131d22dd7458a at 1731827788279Finished memstore snapshotting TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731827788279Flushing stores of TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a. at 1731827788279Flushing c9fb4027559a6d665f6131d22dd7458a/A: creating writer at 1731827788279Flushing mob file c9fb4027559a6d665f6131d22dd7458a/A: appending metadata at 1731827788283 (+4 ms)Flushing mob file c9fb4027559a6d665f6131d22dd7458a/A: closing flushed file at 1731827788283Flushing c9fb4027559a6d665f6131d22dd7458a/A: appending metadata at 1731827788691 (+408 ms)Flushing c9fb4027559a6d665f6131d22dd7458a/A: closing flushed file at 1731827788691Flushing c9fb4027559a6d665f6131d22dd7458a/B: creating writer at 1731827789097 (+406 ms)Flushing c9fb4027559a6d665f6131d22dd7458a/B: appending metadata at 1731827789099 (+2 ms)Flushing c9fb4027559a6d665f6131d22dd7458a/B: closing flushed file at 1731827789099Flushing c9fb4027559a6d665f6131d22dd7458a/C: creating writer at 1731827789506 (+407 ms)Flushing c9fb4027559a6d665f6131d22dd7458a/C: appending metadata at 1731827789508 (+2 ms)Flushing c9fb4027559a6d665f6131d22dd7458a/C: closing flushed file at 1731827789508Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42a95a6a: reopening flushed file at 1731827789915 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@673f3672: reopening flushed file at 1731827789918 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66f4886: reopening flushed file at 1731827789922 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for c9fb4027559a6d665f6131d22dd7458a in 1651ms, sequenceid=367, compaction requested=true at 1731827789929 (+7 ms)Writing region close event to WAL at 1731827790002 (+73 ms)Running coprocessor post-close hooks at 1731827790006 (+4 ms)Closed at 1731827790006 2024-11-17T07:16:30,009 INFO [RS_CLOSE_REGION-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,009 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=c9fb4027559a6d665f6131d22dd7458a, regionState=CLOSED 2024-11-17T07:16:30,010 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 because future has completed 2024-11-17T07:16:30,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-17T07:16:30,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure c9fb4027559a6d665f6131d22dd7458a, server=1dd62e24b303,36419,1731827614816 in 1.8850 sec 2024-11-17T07:16:30,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-17T07:16:30,015 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c9fb4027559a6d665f6131d22dd7458a, UNASSIGN in 1.8890 sec 2024-11-17T07:16:30,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-17T07:16:30,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8920 sec 2024-11-17T07:16:30,018 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731827790018"}]},"ts":"1731827790018"} 2024-11-17T07:16:30,019 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-17T07:16:30,020 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-17T07:16:30,021 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9010 sec 2024-11-17T07:16:30,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-17T07:16:30,267 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:16:30,268 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-17T07:16:30,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:16:30,269 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:16:30,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T07:16:30,270 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:16:30,272 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,274 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C, FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/recovered.edits] 2024-11-17T07:16:30,277 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/35dabe9548804a6d9ff5fe5b584f37e4 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/35dabe9548804a6d9ff5fe5b584f37e4 2024-11-17T07:16:30,278 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/76fc7ecdbb2a4855a53a7e0e18f8bf4d to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/76fc7ecdbb2a4855a53a7e0e18f8bf4d 2024-11-17T07:16:30,279 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/c27baf56a4194c66a60c3b2688375de5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/A/c27baf56a4194c66a60c3b2688375de5 2024-11-17T07:16:30,280 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/064171b6ef0d44a1a255e4d6c6963ec5 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/064171b6ef0d44a1a255e4d6c6963ec5 2024-11-17T07:16:30,281 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e98db248f0fc4836a396a9a2f0f3b8f1 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/e98db248f0fc4836a396a9a2f0f3b8f1 2024-11-17T07:16:30,282 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/f87aca7811f24882a552a9cf2690f155 to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/B/f87aca7811f24882a552a9cf2690f155 2024-11-17T07:16:30,284 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/7f83601b98e54cc9a10fc36264d8e5db to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/7f83601b98e54cc9a10fc36264d8e5db 2024-11-17T07:16:30,285 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/a5afd0a54d4e4cf390a639b7f681e5fd to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/a5afd0a54d4e4cf390a639b7f681e5fd 2024-11-17T07:16:30,286 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/c1bb939d9be54767a551c1f804ac065b to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/C/c1bb939d9be54767a551c1f804ac065b 2024-11-17T07:16:30,288 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/recovered.edits/370.seqid to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a/recovered.edits/370.seqid 2024-11-17T07:16:30,289 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/default/TestAcidGuarantees/c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,289 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-17T07:16:30,289 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T07:16:30,290 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-17T07:16:30,292 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117016ef61065544c23ba3137d022f83dfb_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117016ef61065544c23ba3137d022f83dfb_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,293 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111702db1890f3904a7db69d40d50a99b1ab_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111702db1890f3904a7db69d40d50a99b1ab_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,294 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170d40e2b5fc3745429f2083c871ad7c5d_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411170d40e2b5fc3745429f2083c871ad7c5d_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,295 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111730dc8d0f8b774a7fb0fdc89f4d874d26_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111730dc8d0f8b774a7fb0fdc89f4d874d26_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,296 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174b7f49100bf44b7da24ede4c85962be3_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411174b7f49100bf44b7da24ede4c85962be3_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,297 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111757afe2cf83f34ae8b795f1bca60b9b46_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111757afe2cf83f34ae8b795f1bca60b9b46_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,298 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117628072ddb9bf4af489f68a8ef84104a6_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117628072ddb9bf4af489f68a8ef84104a6_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,299 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176e5348aff75b4ac982f71be12f6ed880_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411176e5348aff75b4ac982f71be12f6ed880_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,300 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117702116acee6b4e37a00db89b4f184ce0_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117702116acee6b4e37a00db89b4f184ce0_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,301 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111793ac88df29b54b5f8707e3e0aa5eb0e5_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111793ac88df29b54b5f8707e3e0aa5eb0e5_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,301 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179beaf38d10954eb68f40508383b17ce7_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411179beaf38d10954eb68f40508383b17ce7_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,302 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117be7a2488c94b40b69bdbe204e81de460_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117be7a2488c94b40b69bdbe204e81de460_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,303 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cea7838862d04ed49de9cd9234a2e4cc_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cea7838862d04ed49de9cd9234a2e4cc_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,304 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cea7e9571c124a28bfae042176f1b9b8_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117cea7e9571c124a28bfae042176f1b9b8_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,305 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d05f005c59844e029dc57253d2835dd4_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d05f005c59844e029dc57253d2835dd4_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,306 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d725b5dab9864e5bbac67aa40e96a9c6_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117d725b5dab9864e5bbac67aa40e96a9c6_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,306 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e7d79f93d4f64981b957e3258b674780_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117e7d79f93d4f64981b957e3258b674780_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,307 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f95c982a18e843a6bca8e3c9d80bd5d4_c9fb4027559a6d665f6131d22dd7458a to hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241117f95c982a18e843a6bca8e3c9d80bd5d4_c9fb4027559a6d665f6131d22dd7458a 2024-11-17T07:16:30,308 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-17T07:16:30,311 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:16:30,314 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-17T07:16:30,316 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-17T07:16:30,316 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:16:30,316 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-17T07:16:30,317 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731827790316"}]},"ts":"9223372036854775807"} 2024-11-17T07:16:30,319 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-17T07:16:30,319 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c9fb4027559a6d665f6131d22dd7458a, NAME => 'TestAcidGuarantees,,1731827758906.c9fb4027559a6d665f6131d22dd7458a.', STARTKEY => '', ENDKEY => ''}] 2024-11-17T07:16:30,319 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-17T07:16:30,319 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731827790319"}]},"ts":"9223372036854775807"} 2024-11-17T07:16:30,321 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-17T07:16:30,321 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-17T07:16:30,322 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-17T07:16:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-17T07:16:30,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-17T07:16:30,377 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-17T07:16:30,391 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 249), OpenFileDescriptor=451 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=343 (was 393), ProcessCount=11 (was 11), AvailableMemoryMB=1010 (was 1028) 2024-11-17T07:16:30,391 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T07:16:30,391 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T07:16:30,391 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T07:16:30,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:30,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:30,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T07:16:30,391 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T07:16:30,391 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2127823695, stopped=false 2024-11-17T07:16:30,392 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=1dd62e24b303,36491,1731827613877 2024-11-17T07:16:30,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T07:16:30,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T07:16:30,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:16:30,394 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:16:30,394 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T07:16:30,394 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T07:16:30,394 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T07:16:30,394 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T07:16:30,395 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:30,395 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '1dd62e24b303,36419,1731827614816' ***** 2024-11-17T07:16:30,395 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T07:16:30,396 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T07:16:30,396 INFO [RS:0;1dd62e24b303:36419 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T07:16:30,396 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T07:16:30,396 INFO [RS:0;1dd62e24b303:36419 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T07:16:30,396 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(959): stopping server 1dd62e24b303,36419,1731827614816 2024-11-17T07:16:30,396 INFO [RS:0;1dd62e24b303:36419 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T07:16:30,396 INFO [RS:0;1dd62e24b303:36419 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;1dd62e24b303:36419. 2024-11-17T07:16:30,396 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T07:16:30,396 DEBUG [RS:0;1dd62e24b303:36419 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T07:16:30,396 DEBUG [RS:0;1dd62e24b303:36419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:30,397 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T07:16:30,397 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T07:16:30,397 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T07:16:30,397 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T07:16:30,398 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T07:16:30,398 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-17T07:16:30,398 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T07:16:30,398 INFO [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T07:16:30,398 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T07:16:30,398 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T07:16:30,398 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T07:16:30,398 INFO [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-17T07:16:30,398 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T07:16:30,421 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/.tmp/info/c7fae2e3d3124cb59d3e5d55ed396079 is 95, key is TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f./info:/1731827645068/DeleteFamily/seqid=0 2024-11-17T07:16:30,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742486_1662 (size=6514) 2024-11-17T07:16:30,495 INFO [regionserver/1dd62e24b303:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T07:16:30,598 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T07:16:30,599 INFO [regionserver/1dd62e24b303:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-17T07:16:30,599 INFO [regionserver/1dd62e24b303:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-17T07:16:30,799 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T07:16:30,825 INFO [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/.tmp/info/c7fae2e3d3124cb59d3e5d55ed396079 2024-11-17T07:16:30,843 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/.tmp/ns/d44ee1e4e5994019a476a5160d2aadaf is 93, key is TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f./ns:/1731827645023/DeleteFamily/seqid=0 2024-11-17T07:16:30,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742487_1663 (size=5906) 2024-11-17T07:16:30,999 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T07:16:31,199 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T07:16:31,246 INFO [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/.tmp/ns/d44ee1e4e5994019a476a5160d2aadaf 2024-11-17T07:16:31,270 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/.tmp/rep_barrier/b5e07b3cd2e54da4a39e53987baab28d is 102, key is TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f./rep_barrier:/1731827645023/DeleteFamily/seqid=0 2024-11-17T07:16:31,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742488_1664 (size=6025) 2024-11-17T07:16:31,399 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T07:16:31,599 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T07:16:31,674 INFO [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/.tmp/rep_barrier/b5e07b3cd2e54da4a39e53987baab28d 2024-11-17T07:16:31,696 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/.tmp/table/9b75c195758e42f2a98dc74094488eda is 96, key is TestAcidGuarantees,,1731827618654.cccbc0d9c2f0bc3781663c8c9bb7223f./table:/1731827645023/DeleteFamily/seqid=0 2024-11-17T07:16:31,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742489_1665 (size=5936) 2024-11-17T07:16:31,800 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T07:16:32,000 DEBUG [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T07:16:32,104 INFO [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/.tmp/table/9b75c195758e42f2a98dc74094488eda 2024-11-17T07:16:32,107 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/.tmp/info/c7fae2e3d3124cb59d3e5d55ed396079 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/info/c7fae2e3d3124cb59d3e5d55ed396079 2024-11-17T07:16:32,109 INFO [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/info/c7fae2e3d3124cb59d3e5d55ed396079, entries=12, sequenceid=90, filesize=6.4 K 2024-11-17T07:16:32,110 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/.tmp/ns/d44ee1e4e5994019a476a5160d2aadaf as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/ns/d44ee1e4e5994019a476a5160d2aadaf 2024-11-17T07:16:32,112 INFO [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/ns/d44ee1e4e5994019a476a5160d2aadaf, entries=8, sequenceid=90, filesize=5.8 K 2024-11-17T07:16:32,112 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/.tmp/rep_barrier/b5e07b3cd2e54da4a39e53987baab28d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/rep_barrier/b5e07b3cd2e54da4a39e53987baab28d 2024-11-17T07:16:32,115 INFO [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/rep_barrier/b5e07b3cd2e54da4a39e53987baab28d, entries=6, sequenceid=90, filesize=5.9 K 2024-11-17T07:16:32,115 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/.tmp/table/9b75c195758e42f2a98dc74094488eda as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/table/9b75c195758e42f2a98dc74094488eda 2024-11-17T07:16:32,117 INFO [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/table/9b75c195758e42f2a98dc74094488eda, entries=7, sequenceid=90, filesize=5.8 K 2024-11-17T07:16:32,118 INFO [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1720ms, sequenceid=90, compaction requested=false 2024-11-17T07:16:32,122 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-17T07:16:32,122 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T07:16:32,122 INFO [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T07:16:32,122 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731827790398Running coprocessor pre-close hooks at 1731827790398Disabling compacts and flushes for region at 1731827790398Disabling writes for close at 1731827790398Obtaining lock to block concurrent updates at 1731827790398Preparing flush snapshotting stores in 1588230740 at 1731827790398Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731827790399 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731827790399Flushing 1588230740/info: creating writer at 1731827790399Flushing 1588230740/info: appending metadata at 1731827790421 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731827790421Flushing 1588230740/ns: creating writer at 1731827790829 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731827790842 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731827790842Flushing 1588230740/rep_barrier: creating writer at 1731827791250 (+408 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731827791270 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731827791270Flushing 1588230740/table: creating writer at 1731827791678 (+408 ms)Flushing 1588230740/table: appending metadata at 1731827791696 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731827791696Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@367b4e1f: reopening flushed file at 1731827792107 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3dddba4f: reopening flushed file at 1731827792110 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f45ea3a: reopening flushed file at 1731827792112 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74ae5a61: reopening flushed file at 1731827792115 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1720ms, sequenceid=90, compaction requested=false at 1731827792118 (+3 ms)Writing region close event to WAL at 1731827792119 (+1 ms)Running coprocessor post-close hooks at 1731827792122 (+3 ms)Closed at 1731827792122 2024-11-17T07:16:32,122 DEBUG [RS_CLOSE_META-regionserver/1dd62e24b303:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T07:16:32,200 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(976): stopping server 1dd62e24b303,36419,1731827614816; all regions closed. 2024-11-17T07:16:32,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741834_1010 (size=25344) 2024-11-17T07:16:32,206 DEBUG [RS:0;1dd62e24b303:36419 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/oldWALs 2024-11-17T07:16:32,206 INFO [RS:0;1dd62e24b303:36419 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1dd62e24b303%2C36419%2C1731827614816.meta:.meta(num 1731827618230) 2024-11-17T07:16:32,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741833_1009 (size=19191706) 2024-11-17T07:16:32,209 DEBUG [RS:0;1dd62e24b303:36419 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/oldWALs 2024-11-17T07:16:32,209 INFO [RS:0;1dd62e24b303:36419 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 1dd62e24b303%2C36419%2C1731827614816:(num 1731827617774) 2024-11-17T07:16:32,209 DEBUG [RS:0;1dd62e24b303:36419 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T07:16:32,209 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T07:16:32,209 INFO [RS:0;1dd62e24b303:36419 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T07:16:32,209 INFO [RS:0;1dd62e24b303:36419 {}] hbase.ChoreService(370): Chore service for: regionserver/1dd62e24b303:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-17T07:16:32,210 INFO [RS:0;1dd62e24b303:36419 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T07:16:32,210 INFO [regionserver/1dd62e24b303:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T07:16:32,210 INFO [RS:0;1dd62e24b303:36419 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36419 2024-11-17T07:16:32,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T07:16:32,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/1dd62e24b303,36419,1731827614816 2024-11-17T07:16:32,214 INFO [RS:0;1dd62e24b303:36419 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T07:16:32,222 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [1dd62e24b303,36419,1731827614816] 2024-11-17T07:16:32,224 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/1dd62e24b303,36419,1731827614816 already deleted, retry=false 2024-11-17T07:16:32,224 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 1dd62e24b303,36419,1731827614816 expired; onlineServers=0 2024-11-17T07:16:32,225 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '1dd62e24b303,36491,1731827613877' ***** 2024-11-17T07:16:32,225 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T07:16:32,225 INFO [M:0;1dd62e24b303:36491 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T07:16:32,225 INFO [M:0;1dd62e24b303:36491 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T07:16:32,225 DEBUG [M:0;1dd62e24b303:36491 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T07:16:32,225 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T07:16:32,225 DEBUG [M:0;1dd62e24b303:36491 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T07:16:32,225 DEBUG [master/1dd62e24b303:0:becomeActiveMaster-HFileCleaner.small.0-1731827617454 {}] cleaner.HFileCleaner(306): Exit Thread[master/1dd62e24b303:0:becomeActiveMaster-HFileCleaner.small.0-1731827617454,5,FailOnTimeoutGroup] 2024-11-17T07:16:32,225 DEBUG [master/1dd62e24b303:0:becomeActiveMaster-HFileCleaner.large.0-1731827617453 {}] cleaner.HFileCleaner(306): Exit Thread[master/1dd62e24b303:0:becomeActiveMaster-HFileCleaner.large.0-1731827617453,5,FailOnTimeoutGroup] 2024-11-17T07:16:32,225 INFO [M:0;1dd62e24b303:36491 {}] hbase.ChoreService(370): Chore service for: master/1dd62e24b303:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T07:16:32,225 INFO [M:0;1dd62e24b303:36491 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T07:16:32,225 DEBUG [M:0;1dd62e24b303:36491 {}] master.HMaster(1795): Stopping service threads 2024-11-17T07:16:32,225 INFO [M:0;1dd62e24b303:36491 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T07:16:32,225 INFO [M:0;1dd62e24b303:36491 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T07:16:32,226 ERROR [M:0;1dd62e24b303:36491 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (2068722025) connection to localhost/127.0.0.1:44301 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:44301,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-17T07:16:32,226 INFO [M:0;1dd62e24b303:36491 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T07:16:32,226 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T07:16:32,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T07:16:32,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T07:16:32,227 DEBUG [M:0;1dd62e24b303:36491 {}] zookeeper.ZKUtil(347): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T07:16:32,227 WARN [M:0;1dd62e24b303:36491 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T07:16:32,228 INFO [M:0;1dd62e24b303:36491 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/.lastflushedseqids 2024-11-17T07:16:32,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742490_1666 (size=72) 2024-11-17T07:16:32,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T07:16:32,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36419-0x10124c627cf0001, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T07:16:32,323 INFO [RS:0;1dd62e24b303:36419 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T07:16:32,324 INFO [RS:0;1dd62e24b303:36419 {}] regionserver.HRegionServer(1031): Exiting; stopping=1dd62e24b303,36419,1731827614816; zookeeper connection closed. 2024-11-17T07:16:32,324 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ec2fc5b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ec2fc5b 2024-11-17T07:16:32,324 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-17T07:16:32,637 INFO [M:0;1dd62e24b303:36491 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T07:16:32,637 INFO [M:0;1dd62e24b303:36491 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T07:16:32,637 DEBUG [M:0;1dd62e24b303:36491 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T07:16:32,637 INFO [M:0;1dd62e24b303:36491 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T07:16:32,637 DEBUG [M:0;1dd62e24b303:36491 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T07:16:32,637 DEBUG [M:0;1dd62e24b303:36491 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T07:16:32,637 DEBUG [M:0;1dd62e24b303:36491 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T07:16:32,637 INFO [M:0;1dd62e24b303:36491 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=808.54 KB heapSize=992.98 KB 2024-11-17T07:16:32,652 DEBUG [M:0;1dd62e24b303:36491 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6ba8187f63254b189be287f5d6380d7c is 82, key is hbase:meta,,1/info:regioninfo/1731827618312/Put/seqid=0 2024-11-17T07:16:32,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742491_1667 (size=5672) 2024-11-17T07:16:32,932 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-17T07:16:33,056 INFO [M:0;1dd62e24b303:36491 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2292 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6ba8187f63254b189be287f5d6380d7c 2024-11-17T07:16:33,076 DEBUG [M:0;1dd62e24b303:36491 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1cb51fb671c44c4497ce35ab2ce49868 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1731827761970/Put/seqid=0 2024-11-17T07:16:33,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742492_1668 (size=45073) 2024-11-17T07:16:33,480 INFO [M:0;1dd62e24b303:36491 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=807.98 KB at sequenceid=2292 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1cb51fb671c44c4497ce35ab2ce49868 2024-11-17T07:16:33,483 INFO [M:0;1dd62e24b303:36491 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1cb51fb671c44c4497ce35ab2ce49868 2024-11-17T07:16:33,497 DEBUG [M:0;1dd62e24b303:36491 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/89ec2da601454e38ba6a7ce080e4394d is 69, key is 1dd62e24b303,36419,1731827614816/rs:state/1731827617501/Put/seqid=0 2024-11-17T07:16:33,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073742493_1669 (size=5156) 2024-11-17T07:16:33,901 INFO [M:0;1dd62e24b303:36491 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2292 (bloomFilter=true), to=hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/89ec2da601454e38ba6a7ce080e4394d 2024-11-17T07:16:33,904 DEBUG [M:0;1dd62e24b303:36491 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6ba8187f63254b189be287f5d6380d7c as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6ba8187f63254b189be287f5d6380d7c 2024-11-17T07:16:33,906 INFO [M:0;1dd62e24b303:36491 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6ba8187f63254b189be287f5d6380d7c, entries=8, sequenceid=2292, filesize=5.5 K 2024-11-17T07:16:33,907 DEBUG [M:0;1dd62e24b303:36491 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/1cb51fb671c44c4497ce35ab2ce49868 as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1cb51fb671c44c4497ce35ab2ce49868 2024-11-17T07:16:33,909 INFO [M:0;1dd62e24b303:36491 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1cb51fb671c44c4497ce35ab2ce49868 2024-11-17T07:16:33,909 INFO [M:0;1dd62e24b303:36491 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/1cb51fb671c44c4497ce35ab2ce49868, entries=168, sequenceid=2292, filesize=44.0 K 2024-11-17T07:16:33,910 DEBUG [M:0;1dd62e24b303:36491 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/89ec2da601454e38ba6a7ce080e4394d as hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/89ec2da601454e38ba6a7ce080e4394d 2024-11-17T07:16:33,912 INFO [M:0;1dd62e24b303:36491 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44301/user/jenkins/test-data/42bb872c-f780-d26c-eb2c-1b687441f650/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/89ec2da601454e38ba6a7ce080e4394d, entries=1, sequenceid=2292, filesize=5.0 K 2024-11-17T07:16:33,913 INFO [M:0;1dd62e24b303:36491 {}] regionserver.HRegion(3140): Finished flush of dataSize ~808.54 KB/827941, heapSize ~992.68 KB/1016504, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2292, compaction requested=false 2024-11-17T07:16:33,916 INFO [M:0;1dd62e24b303:36491 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T07:16:33,916 DEBUG [M:0;1dd62e24b303:36491 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731827792637Disabling compacts and flushes for region at 1731827792637Disabling writes for close at 1731827792637Obtaining lock to block concurrent updates at 1731827792637Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731827792637Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=827941, getHeapSize=1016744, getOffHeapSize=0, getCellsCount=2485 at 1731827792638 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731827792638Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731827792638Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731827792652 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731827792652Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731827793059 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731827793075 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731827793075Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731827793483 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731827793497 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731827793497Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@486c4a03: reopening flushed file at 1731827793903 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2490166b: reopening flushed file at 1731827793906 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fe59a97: reopening flushed file at 1731827793909 (+3 ms)Finished flush of dataSize ~808.54 KB/827941, heapSize ~992.68 KB/1016504, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1276ms, sequenceid=2292, compaction requested=false at 1731827793913 (+4 ms)Writing region close event to WAL at 1731827793916 (+3 ms)Closed at 1731827793916 2024-11-17T07:16:33,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46705 is added to blk_1073741830_1006 (size=978112) 2024-11-17T07:16:33,919 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T07:16:33,919 INFO [M:0;1dd62e24b303:36491 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T07:16:33,920 INFO [M:0;1dd62e24b303:36491 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36491 2024-11-17T07:16:33,920 INFO [M:0;1dd62e24b303:36491 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T07:16:34,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T07:16:34,022 INFO [M:0;1dd62e24b303:36491 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T07:16:34,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36491-0x10124c627cf0000, quorum=127.0.0.1:56533, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T07:16:34,027 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7acb84b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-17T07:16:34,029 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T07:16:34,029 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T07:16:34,030 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T07:16:34,030 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/hadoop.log.dir/,STOPPED} 2024-11-17T07:16:34,033 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-17T07:16:34,033 WARN [BP-711933474-172.17.0.2-1731827610116 heartbeating to localhost/127.0.0.1:44301 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-17T07:16:34,033 WARN [BP-711933474-172.17.0.2-1731827610116 heartbeating to localhost/127.0.0.1:44301 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-711933474-172.17.0.2-1731827610116 (Datanode Uuid 2c01d2b6-9a80-45f3-9f18-75dcd942ec6c) service to localhost/127.0.0.1:44301 2024-11-17T07:16:34,033 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-17T07:16:34,035 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/cluster_ec49d093-dc1d-39e4-b51d-2529c25f0961/data/data1/current/BP-711933474-172.17.0.2-1731827610116 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T07:16:34,035 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/cluster_ec49d093-dc1d-39e4-b51d-2529c25f0961/data/data2/current/BP-711933474-172.17.0.2-1731827610116 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-17T07:16:34,037 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-17T07:16:34,045 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-17T07:16:34,046 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-17T07:16:34,046 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-17T07:16:34,046 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-17T07:16:34,046 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d66ddbe5-7b20-a974-b77d-9b51c6e76b9d/hadoop.log.dir/,STOPPED} 2024-11-17T07:16:34,062 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T07:16:34,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-17T07:16:34,387 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T07:16:34,388 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-17T07:16:34,388 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-17T07:16:34,388 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees